From 1e0386e9e681c0217f11428f210f82f29e433c7d Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 2 Nov 2021 15:52:42 +0100 Subject: [PATCH] Fix `ip` matcher lexer to differentiate filter from identifier (#4598) * Fix `ip` matcher lexer to differentiate filter from identifier Signed-off-by: Kaviraj * Revert small fmt fix on expr.y Signed-off-by: Kaviraj * PR remarks from Cyril Signed-off-by: Kaviraj * Add ip line filter parser test in mix with ip as label Signed-off-by: Kaviraj --- pkg/logql/lex.go | 9 +++++- pkg/logql/lex_test.go | 6 ++++ pkg/logql/parser_test.go | 70 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index 10fe28d21bb42..9002ac478c5f7 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -107,6 +107,9 @@ var functionTokens = map[string]int{ OpConvBytes: BYTES_CONV, OpConvDuration: DURATION_CONV, OpConvDurationSeconds: DURATION_SECONDS_CONV, + + // filterOp + OpFilterIP: IP, } type lexer struct { @@ -193,7 +196,11 @@ func (l *lexer) Lex(lval *exprSymType) int { } } - if tok, ok := functionTokens[tokenText]; ok && isFunction(l.Scanner) { + if tok, ok := functionTokens[tokenText]; ok { + if !isFunction(l.Scanner) { + lval.str = tokenText + return IDENTIFIER + } return tok } diff --git a/pkg/logql/lex_test.go b/pkg/logql/lex_test.go index 450e958676e5e..b57af7969f1bf 100644 --- a/pkg/logql/lex_test.go +++ b/pkg/logql/lex_test.go @@ -52,6 +52,12 @@ func TestLex(t *testing.T) { {`{foo="bar"} #|~ "\\w+"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, {`#{foo="bar"} |~ "\\w+"`, []int{}}, {`{foo="#"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, + {`{foo="bar"}|logfmt|ip="b"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, STRING}}, + {`{foo="bar"}|logfmt|rate="b"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, STRING}}, + {`{foo="bar"}|logfmt|b=ip("b")`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, IP, OPEN_PARENTHESIS, STRING, CLOSE_PARENTHESIS}}, + {`{foo="bar"}|logfmt|=ip("b")`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE_EXACT, IP, OPEN_PARENTHESIS, STRING, CLOSE_PARENTHESIS}}, + {`ip`, []int{IDENTIFIER}}, + {`rate`, []int{IDENTIFIER}}, {`{foo="bar"} | json | baz="#"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON, PIPE, IDENTIFIER, EQ, STRING}}, {`{foo="bar"} # |~ "\\w+" diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index a14268534ab51..2a33654be531f 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -139,6 +139,16 @@ func TestParse(t *testing.T) { Operation: "rate", }, }, + { + in: `{ foo = "bar" }|logfmt|rate="a"`, // rate should also be able to use it as IDENTIFIER + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "rate", "a"))), + }, + ), + }, { in: `rate({ foo = "bar" }[5d])`, exp: &RangeAggregationExpr{ @@ -351,6 +361,20 @@ func TestParse(t *testing.T) { }, ), }, + { + in: `{ foo = "bar" , ip="foo"}|logfmt|= ip("127.0.0.1")|ip="2.3.4.5"|ip="abc"|ipaddr=ip("4.5.6.7")|ip=ip("6.7.8.9")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar"), mustNewMatcher(labels.MatchEqual, "ip", "foo")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLineFilterExpr(labels.MatchEqual, OpFilterIP, "127.0.0.1"), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "abc"))), + newLabelFilterExpr(log.NewIPLabelFilter("4.5.6.7", "ipaddr", log.LabelFilterEqual)), + newLabelFilterExpr(log.NewIPLabelFilter("6.7.8.9", "ip", log.LabelFilterEqual)), + }, + ), + }, { in: `{foo="bar"} |= ip("123.123.123.123")`, exp: newPipelineExpr( @@ -524,6 +548,52 @@ func TestParse(t *testing.T) { }, ), }, + { + in: `{ foo = "bar" }|logfmt|ip="2.3.4.5"`, // just using `ip` as a label name(identifier) should work + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))), + }, + ), + }, + { + in: `{ foo = "bar" }|logfmt|ip="2.3.4.5"|ip="abc"`, // just using `ip` as a label name should work with chaining + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "abc"))), + }, + ), + }, + { + in: `{ foo = "bar" }|logfmt|ip="2.3.4.5"|ip="abc"|ipaddr=ip("4.5.6.7")`, // `ip` should work as both label name and filter in same query + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "abc"))), + newLabelFilterExpr(log.NewIPLabelFilter("4.5.6.7", "ipaddr", log.LabelFilterEqual)), + }, + ), + }, + { + in: `{ foo = "bar" }|logfmt|ip="2.3.4.5"|ip="abc"|ipaddr=ip("4.5.6.7")|ip=ip("6.7.8.9")`, // `ip` should work as both label name and filter in same query with same name. + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "abc"))), + newLabelFilterExpr(log.NewIPLabelFilter("4.5.6.7", "ipaddr", log.LabelFilterEqual)), + newLabelFilterExpr(log.NewIPLabelFilter("6.7.8.9", "ip", log.LabelFilterEqual)), + }, + ), + }, { in: `sum(3 ,count_over_time({ foo = "bar" }[5h]))`, err: logqlmodel.NewParseError("unsupported parameter for operation sum(3,", 0, 0),