From 7e7fe5a941c437eb3174f6299ce09b2422315052 Mon Sep 17 00:00:00 2001 From: tdakkota Date: Fri, 21 Jun 2024 07:47:59 +0300 Subject: [PATCH] perf(chstorage): use `hasToken` to skip blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` │ base │ new │ │ sec/op │ sec/op vs base │ LogQL/Failed_POST_requests 74.24m ± 11% 45.39m ± 10% -38.86% (p=0.000 n=15) │ base │ new │ │ ch-sec/op │ ch-sec/op vs base │ LogQL/Failed_POST_requests 64.22m ± 12% 35.14m ± 12% -45.28% (p=0.000 n=15) │ base │ new │ │ ch-mem-bytes/op │ ch-mem-bytes/op vs base │ LogQL/Failed_POST_requests 61.83Mi ± 2% 41.81Mi ± 10% -32.38% (p=0.000 n=15) │ base │ new │ │ ch-read-bytes/op │ ch-read-bytes/op vs base │ LogQL/Failed_POST_requests 304.29Mi ± 0% 73.85Mi ± 0% -75.73% (p=0.000 n=15) │ base │ new │ │ ch-read-rows/op │ ch-read-rows/op vs base │ LogQL/Failed_POST_requests 1000.1k ± 0% 227.3k ± 0% -77.27% (p=0.000 n=15) ``` --- internal/chstorage/querier_logs_query.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/chstorage/querier_logs_query.go b/internal/chstorage/querier_logs_query.go index 72fabe5e..c0a7597a 100644 --- a/internal/chstorage/querier_logs_query.go +++ b/internal/chstorage/querier_logs_query.go @@ -343,13 +343,25 @@ func (q *Querier) lineFilter(m logql.LineFilter) (e chsql.Expr, rerr error) { switch m.Op { case logql.OpEq, logql.OpNotEq: expr := chsql.Contains("body", m.By.Value) + + // Clickhouse does not use tokenbf_v1 index to skip blocks + // with position* functions for some reason. + // + // Force to skip using hasToken function. + // + // Note that such optimization is applied only if operation is not negated to + // avoid false-negative skipping. + if val := m.By.Value; m.Op != logql.OpNotEq && chsql.IsSingleToken(val) { + expr = chsql.And(expr, + chsql.HasToken(chsql.Ident("body"), val), + ) + } + { // HACK: check for special case of hex-encoded trace_id and span_id. // Like `{http_method=~".+"} |= "af36000000000000c517000000000003"`. // TODO(ernado): also handle regex? - v, _ := hex.DecodeString(m.By.Value) - switch len(v) { case len(otelstorage.TraceID{}): expr = chsql.Or(expr, chsql.Eq(