diff --git a/.drone/drone.yml b/.drone/drone.yml index d56879625ef66..da6b576c73cfa 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -12,28 +12,28 @@ workspace: steps: - name: test - image: grafana/loki-build-image:0.14.0 + image: grafana/loki-build-image:0.15.0 commands: - make BUILD_IN_CONTAINER=false test depends_on: - clone - name: lint - image: grafana/loki-build-image:0.14.0 + image: grafana/loki-build-image:0.15.0 commands: - make BUILD_IN_CONTAINER=false lint depends_on: - clone - name: check-generated-files - image: grafana/loki-build-image:0.14.0 + image: grafana/loki-build-image:0.15.0 commands: - make BUILD_IN_CONTAINER=false check-generated-files depends_on: - clone - name: check-mod - image: grafana/loki-build-image:0.14.0 + image: grafana/loki-build-image:0.15.0 commands: - make BUILD_IN_CONTAINER=false check-mod depends_on: diff --git a/.golangci.yml b/.golangci.yml index 0c0873b842152..404bdd878a9ff 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,6 +30,9 @@ run: # no need to include all autogenerated files, we confidently recognize # autogenerated files. If it's not please let us know. skip-files: + - .*.pb.go + - .*.y.go + - .*.rl.go # output configuration options output: # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" diff --git a/Makefile b/Makefile index 5d4f8f9b62cee..7c7f3bcadf2a1 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image .PHONY: bigtable-backup, push-bigtable-backup .PHONY: benchmark-store, drone, check-mod -.PHONY: migrate migrate-image lint-markdown +.PHONY: migrate migrate-image lint-markdown ragel SHELL = /usr/bin/env bash @@ -38,7 +38,7 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) # make BUILD_IN_CONTAINER=false target # or you can override this with an environment variable BUILD_IN_CONTAINER ?= true -BUILD_IMAGE_VERSION := 0.14.0 +BUILD_IMAGE_VERSION := 0.15.0 # Docker image info IMAGE_PREFIX ?= grafana @@ -87,6 +87,10 @@ PROTO_GOS := $(patsubst %.proto,%.pb.go,$(PROTO_DEFS)) YACC_DEFS := $(shell find . $(DONT_FIND) -type f -name *.y -print) YACC_GOS := $(patsubst %.y,%.y.go,$(YACC_DEFS)) +# Ragel Files +RAGEL_DEFS := $(shell find . $(DONT_FIND) -type f -name *.rl -print) +RAGEL_GOS := $(patsubst %.rl,%.rl.go,$(RAGEL_DEFS)) + # Promtail UI files PROMTAIL_GENERATED_FILE := clients/pkg/promtail/server/ui/assets_vfsdata.go PROMTAIL_UI_FILES := $(shell find ./clients/pkg/promtail/server/ui -type f -name assets_vfsdata.go -prune -o -print) @@ -126,8 +130,8 @@ binfmt: all: promtail logcli loki loki-canary check-generated-files # This is really a check for the CI to make sure generated files are built and checked in manually -check-generated-files: touch-protobuf-sources yacc protos clients/pkg/promtail/server/ui/assets_vfsdata.go - @if ! (git diff --exit-code $(YACC_GOS) $(PROTO_GOS) $(PROMTAIL_GENERATED_FILE)); then \ +check-generated-files: touch-protobuf-sources yacc ragel protos clients/pkg/promtail/server/ui/assets_vfsdata.go + @if ! (git diff --exit-code $(YACC_GOS) $(RAGEL_GOS) $(PROTO_GOS) $(PROMTAIL_GENERATED_FILE)); then \ echo "\nChanges found in generated files"; \ echo "Run 'make check-generated-files' and commit the changes to fix this error."; \ echo "If you are actively developing these files you can ignore this error"; \ @@ -147,7 +151,7 @@ touch-protobuf-sources: # Logcli # ########## -logcli: yacc cmd/logcli/logcli +logcli: yacc ragel cmd/logcli/logcli logcli-image: $(SUDO) docker build -t $(IMAGE_PREFIX)/logcli:$(IMAGE_TAG) -f cmd/logcli/Dockerfile . @@ -160,8 +164,8 @@ cmd/logcli/logcli: $(APP_GO_FILES) cmd/logcli/main.go # Loki # ######## -loki: protos yacc cmd/loki/loki -loki-debug: protos yacc cmd/loki/loki-debug +loki: protos yacc ragel cmd/loki/loki +loki-debug: protos yacc ragel cmd/loki/loki-debug cmd/loki/loki: $(APP_GO_FILES) cmd/loki/main.go CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) @@ -175,7 +179,7 @@ cmd/loki/loki-debug: $(APP_GO_FILES) cmd/loki/main.go # Loki-Canary # ############### -loki-canary: protos yacc cmd/loki-canary/loki-canary +loki-canary: protos yacc ragel cmd/loki-canary/loki-canary cmd/loki-canary/loki-canary: $(APP_GO_FILES) cmd/loki-canary/main.go CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) @@ -206,8 +210,8 @@ PROMTAIL_DEBUG_GO_FLAGS = $(DYN_DEBUG_GO_FLAGS) endif endif -promtail: yacc clients/cmd/promtail/promtail -promtail-debug: yacc clients/cmd/promtail/promtail-debug +promtail: yacc ragel clients/cmd/promtail/promtail +promtail-debug: yacc ragel clients/cmd/promtail/promtail-debug promtail-clean-assets: rm -rf clients/pkg/promtail/server/ui/assets_vfsdata.go @@ -308,6 +312,25 @@ else rm ${@}.back endif +######### +# Ragels # +######### + +ragel: $(RAGEL_GOS) + +%.rl.go: %.rl +ifeq ($(BUILD_IN_CONTAINER),true) + @mkdir -p $(shell pwd)/.pkg + @mkdir -p $(shell pwd)/.cache + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ + -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ + -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; +else + ragel -Z $< -o $@ +endif + ############# # Protobufs # ############# diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index f09ab8e7f547b..ea9f724a49e1b 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -154,9 +154,10 @@ In case of errors, for instance if the line is not in the expected format, the l If an extracted label key name already exists in the original log stream, the extracted label key will be suffixed with the `_extracted` keyword to make the distinction between the two labels. You can forcefully override the original label using a [label formatter expression](#labels-format-expression). However if an extracted key appears twice, only the latest label value will be kept. -We support currently support [json](#json), [logfmt](#logfmt), [regexp](#regexp) and [unpack](#unpack) parsers. +Loki supports [JSON](#json), [logfmt](#logfmt), [pattern](#pattern), [regexp](#regexp) and [unpack](#unpack) parsers. -It's easier to use the predefined parsers like `json` and `logfmt` when you can, falling back to `regexp` when the log lines have unusual structure. Multiple parsers can be used during the same log pipeline which is useful when you want to parse complex logs. ([see examples](#multiple-parsers)) +It's easier to use the predefined parsers `json` and `logfmt` when you can. If you can't, the `pattern` and `regexp` parsers can be used for log lines with an unusual structure. The `pattern` parser is easier and faster to write; it also outperforms the `regexp` parser. +Multiple parsers can be used by a single log pipeline. This is useful for parsing complex logs. There are examples in [Multiple parsers](#multiple-parsers). ##### Json @@ -277,6 +278,60 @@ will get those labels extracted: "status" => "200" ``` +##### Pattern + +The pattern parser allows the explicit extraction of fields from log lines by defining a pattern expression. The expression matches the structure of a log line. + +Consider this NGINX log line. + +```log +0.191.12.2 - - [10/Jun/2021:09:14:29 +0000] "GET /api/plugins/versioncheck HTTP/1.1" 200 2 "-" "Go-http-client/2.0" "13.76.247.102, 34.120.177.193" "TLSv1.2" "US" "" +``` + +This log line can be parsed with the expression + +` - - <_> " <_>" <_> "" <_>` + +to extract these fields: + +```kv +"ip" => "0.191.12.2" +"method" => "GET" +"uri" => "/api/plugins/versioncheck" +"status" => "200" +"size" => "2" +"agent" => "Go-http-client/2.0" +``` + +A pattern expression is composed of captures and literals. + +A capture is a field name delimited by the `<` and `>` characters. `` defines the field name `example`. +An unnamed capture appears as `<_>`. The unnamed capture skips matched content. + +Captures are matched from the line beginning or the previous set of literals, to the line end or the next set of literals. +If a capture is not matched, the pattern parser will stop. + +Literals can be any sequence of UTF-8 characters, including whitespace characters. + +By default, a pattern expression is anchored at the start of the log line. If the expression start with literals, then the log line must also start with the same set of literals. Use `<_>` at the beginning of the expression to anchor the expression at the start. + +Consider the log line + +```log +level=debug ts=2021-06-10T09:24:13.472094048Z caller=logging.go:66 traceID=0568b66ad2d9294c msg="POST /loki/api/v1/push (204) 16.652862ms" +``` + +To match `msg="`, use the expression: + +```pattern +<_> msg=" () " +``` + +A pattern expression is invalid if + +- It does not contain any named capture. +- It contains two consecutive captures not separated by whitespace characters. + ##### regexp Unlike the logfmt and json, which extract implicitly all values and takes no parameters, the **regexp** parser takes a single parameter `| regexp ""` which is the regular expression using the [Golang](https://golang.org/) [RE2 syntax](https://github.com/google/re2/wiki/Syntax). diff --git a/loki-build-image/Dockerfile b/loki-build-image/Dockerfile index eb96b56a46140..06d92d84ad8a7 100644 --- a/loki-build-image/Dockerfile +++ b/loki-build-image/Dockerfile @@ -40,7 +40,7 @@ RUN GO111MODULE=on go get github.com/fatih/faillint@v1.5.0 FROM golang:1.16.2-buster RUN apt-get update && \ apt-get install -qy \ - musl gnupg \ + musl gnupg ragel \ file zip unzip jq gettext\ protobuf-compiler libprotobuf-dev \ libsystemd-dev && \ diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index cbee578187c45..4c7256d355a92 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -330,6 +330,8 @@ func (e *labelParserExpr) Stage() (log.Stage, error) { return log.NewRegexpParser(e.param) case OpParserTypeUnpack: return log.NewUnpackParser(), nil + case OpParserTypePattern: + return log.NewPatternParser(e.param) default: return nil, fmt.Errorf("unknown parser operator: %s", e.op) } @@ -601,10 +603,11 @@ const ( OpTypeLTE = "<=" // parsers - OpParserTypeJSON = "json" - OpParserTypeLogfmt = "logfmt" - OpParserTypeRegexp = "regexp" - OpParserTypeUnpack = "unpack" + OpParserTypeJSON = "json" + OpParserTypeLogfmt = "logfmt" + OpParserTypeRegexp = "regexp" + OpParserTypeUnpack = "unpack" + OpParserTypePattern = "pattern" OpFmtLine = "line_format" OpFmtLabel = "label_format" diff --git a/pkg/logql/ast_test.go b/pkg/logql/ast_test.go index f896f8642d727..4773c1642d3f3 100644 --- a/pkg/logql/ast_test.go +++ b/pkg/logql/ast_test.go @@ -28,6 +28,7 @@ func Test_logSelectorExpr_String(t *testing.T) { {`{foo="bar", bar!="baz"} != "bip" !~ ".+bop" | json`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | unpack | foo>5`, true}, + {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | pattern " bar " | foo>5`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b>=10GB`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | regexp "(?Pfoo|bar)"`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | regexp "(?Pfoo|bar)" | ( ( foo<5.01 , bar>20ms ) or foo="bar" ) | line_format "blip{{.boop}}bap" | label_format foo=bar,bar="blip{{.blop}}"`, true}, @@ -69,6 +70,7 @@ func Test_SampleExpr_String(t *testing.T) { `sum(count_over_time({job="mysql"} | json [5m] offset 10m))`, `sum(count_over_time({job="mysql"} | logfmt [5m]))`, `sum(count_over_time({job="mysql"} | logfmt [5m] offset 10m))`, + `sum(count_over_time({job="mysql"} | pattern " bar " | json [5m]))`, `sum(count_over_time({job="mysql"} | unpack | json [5m]))`, `sum(count_over_time({job="mysql"} | regexp "(?Pfoo|bar)" [5m]))`, `sum(count_over_time({job="mysql"} | regexp "(?Pfoo|bar)" [5m] offset 10m))`, @@ -358,6 +360,8 @@ func Test_parserExpr_Parser(t *testing.T) { {"json", OpParserTypeJSON, "", log.NewJSONParser(), false}, {"unpack", OpParserTypeUnpack, "", log.NewUnpackParser(), false}, {"logfmt", OpParserTypeLogfmt, "", log.NewLogfmtParser(), false}, + {"pattern", OpParserTypePattern, " bar ", mustNewPatternParser(" bar "), false}, + {"pattern err", OpParserTypePattern, "bar", nil, true}, {"regexp", OpParserTypeRegexp, "(?Pfoo)", mustNewRegexParser("(?Pfoo)"), false}, {"regexp err ", OpParserTypeRegexp, "foo", nil, true}, } @@ -389,6 +393,14 @@ func mustNewRegexParser(re string) log.Stage { return r } +func mustNewPatternParser(p string) log.Stage { + r, err := log.NewPatternParser(p) + if err != nil { + panic(err) + } + return r +} + func Test_canInjectVectorGrouping(t *testing.T) { tests := []struct { vecOp string diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index c12d67d22ff52..6218794674055 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -100,7 +100,7 @@ import ( OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME MAX_OVER_TIME STDVAR_OVER_TIME STDDEV_OVER_TIME QUANTILE_OVER_TIME BYTES_CONV DURATION_CONV DURATION_SECONDS_CONV - FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME LABEL_REPLACE UNPACK OFFSET + FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME LABEL_REPLACE UNPACK OFFSET PATTERN // Operators are listed with increasing precedence. %left OR @@ -246,6 +246,7 @@ labelParser: | LOGFMT { $$ = newLabelParserExpr(OpParserTypeLogfmt, "") } | REGEXP STRING { $$ = newLabelParserExpr(OpParserTypeRegexp, $2) } | UNPACK { $$ = newLabelParserExpr(OpParserTypeUnpack, "") } + | PATTERN STRING { $$ = newLabelParserExpr(OpParserTypePattern, $2) } ; jsonExpressionParser: diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index fe657cbb9395c..fcafe2b979434 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -1,18 +1,15 @@ // Code generated by goyacc -p expr -o pkg/logql/expr.y.go pkg/logql/expr.y. DO NOT EDIT. -//line pkg/logql/expr.y:2 package logql import __yyfmt__ "fmt" -//line pkg/logql/expr.y:2 import ( "github.com/grafana/loki/pkg/logql/log" "github.com/prometheus/prometheus/pkg/labels" "time" ) -//line pkg/logql/expr.y:12 type exprSymType struct { yys int Expr Expr @@ -118,21 +115,22 @@ const ABSENT_OVER_TIME = 57402 const LABEL_REPLACE = 57403 const UNPACK = 57404 const OFFSET = 57405 -const OR = 57406 -const AND = 57407 -const UNLESS = 57408 -const CMP_EQ = 57409 -const NEQ = 57410 -const LT = 57411 -const LTE = 57412 -const GT = 57413 -const GTE = 57414 -const ADD = 57415 -const SUB = 57416 -const MUL = 57417 -const DIV = 57418 -const MOD = 57419 -const POW = 57420 +const PATTERN = 57406 +const OR = 57407 +const AND = 57408 +const UNLESS = 57409 +const CMP_EQ = 57410 +const NEQ = 57411 +const LT = 57412 +const LTE = 57413 +const GT = 57414 +const GTE = 57415 +const ADD = 57416 +const SUB = 57417 +const MUL = 57418 +const DIV = 57419 +const MOD = 57420 +const POW = 57421 var exprToknames = [...]string{ "$end", @@ -198,6 +196,7 @@ var exprToknames = [...]string{ "LABEL_REPLACE", "UNPACK", "OFFSET", + "PATTERN", "OR", "AND", "UNLESS", @@ -220,9 +219,7 @@ const exprEofCode = 1 const exprErrCode = 2 const exprInitialStackSize = 16 -//line pkg/logql/expr.y:396 -//line yacctab:1 var exprExca = [...]int{ -1, 1, 1, -1, @@ -231,104 +228,100 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 530 +const exprLast = 488 var exprAct = [...]int{ - 229, 183, 75, 4, 165, 58, 154, 5, 159, 192, - 66, 108, 50, 57, 232, 128, 68, 2, 45, 46, + 231, 185, 75, 4, 167, 58, 155, 5, 160, 194, + 66, 109, 50, 57, 234, 129, 68, 2, 45, 46, 47, 48, 49, 50, 71, 42, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, 47, 48, 49, 50, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, - 47, 48, 49, 50, 47, 48, 49, 50, 124, 126, - 127, 64, 237, 97, 167, 126, 127, 101, 62, 63, - 61, 234, 284, 82, 74, 265, 76, 77, 132, 302, - 284, 130, 264, 297, 137, 51, 52, 55, 56, 53, - 54, 45, 46, 47, 48, 49, 50, 287, 138, 232, - 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, - 149, 150, 151, 152, 125, 65, 289, 234, 162, 173, - 168, 171, 172, 169, 170, 267, 268, 269, 76, 77, - 98, 264, 179, 233, 174, 233, 235, 274, 190, 186, - 182, 64, 114, 179, 184, 64, 195, 187, 62, 63, - 271, 272, 62, 63, 256, 238, 156, 194, 244, 114, - 111, 281, 198, 199, 200, 241, 234, 235, 234, 114, - 234, 185, 64, 156, 64, 185, 196, 111, 203, 62, - 63, 62, 63, 227, 230, 188, 236, 111, 239, 130, - 97, 242, 101, 243, 194, 65, 231, 228, 119, 65, - 240, 118, 185, 182, 185, 179, 114, 246, 64, 252, - 250, 300, 248, 193, 64, 62, 63, 296, 157, 155, - 156, 62, 63, 232, 111, 246, 65, 180, 65, 277, - 247, 257, 136, 259, 261, 135, 263, 97, 185, 114, - 64, 262, 273, 258, 185, 134, 97, 62, 63, 275, - 12, 80, 73, 156, 123, 245, 204, 111, 131, 278, - 279, 201, 65, 197, 97, 280, 155, 129, 65, 189, - 60, 282, 283, 181, 121, 12, 205, 288, 202, 295, - 286, 15, 285, 131, 292, 270, 293, 294, 120, 12, - 260, 122, 254, 255, 65, 79, 298, 6, 157, 155, - 78, 19, 20, 33, 34, 36, 37, 35, 38, 39, - 40, 41, 21, 22, 210, 301, 176, 211, 209, 299, - 290, 251, 23, 24, 25, 26, 27, 28, 29, 191, - 249, 178, 30, 31, 32, 18, 207, 12, 175, 208, - 206, 253, 177, 176, 166, 6, 175, 16, 17, 19, - 20, 33, 34, 36, 37, 35, 38, 39, 40, 41, - 21, 22, 225, 163, 222, 226, 224, 223, 221, 161, - 23, 24, 25, 26, 27, 28, 29, 133, 153, 117, - 30, 31, 32, 18, 219, 12, 216, 220, 218, 217, - 215, 70, 291, 6, 72, 16, 17, 19, 20, 33, - 34, 36, 37, 35, 38, 39, 40, 41, 21, 22, - 3, 276, 213, 81, 114, 214, 212, 67, 23, 24, - 25, 26, 27, 28, 29, 160, 72, 166, 30, 31, - 32, 18, 111, 109, 158, 100, 164, 103, 102, 59, - 115, 110, 116, 16, 17, 99, 11, 10, 9, 14, - 104, 106, 105, 8, 112, 113, 237, 83, 84, 85, - 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - 96, 107, 114, 266, 13, 7, 69, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 104, 106, - 105, 0, 112, 113, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, + 47, 48, 49, 50, 47, 48, 49, 50, 125, 127, + 128, 64, 239, 97, 169, 127, 128, 101, 62, 63, + 61, 236, 286, 82, 76, 77, 267, 304, 133, 266, + 286, 131, 266, 299, 138, 51, 52, 55, 56, 53, + 54, 45, 46, 47, 48, 49, 50, 289, 139, 234, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 236, 126, 65, 236, 115, 164, + 175, 170, 173, 174, 171, 172, 269, 270, 271, 248, + 98, 291, 157, 235, 250, 176, 112, 237, 181, 192, + 188, 184, 64, 276, 246, 186, 64, 197, 189, 62, + 63, 64, 274, 62, 63, 181, 240, 248, 62, 63, + 258, 237, 249, 200, 201, 202, 64, 190, 236, 235, + 184, 120, 187, 62, 63, 64, 187, 243, 158, 156, + 119, 187, 62, 63, 273, 229, 232, 181, 238, 115, + 241, 131, 97, 244, 101, 245, 187, 65, 233, 230, + 234, 65, 242, 157, 236, 187, 65, 112, 205, 182, + 64, 254, 252, 115, 115, 196, 64, 62, 63, 115, + 283, 65, 196, 62, 63, 302, 137, 157, 157, 115, + 65, 112, 112, 259, 198, 261, 263, 112, 265, 97, + 187, 195, 136, 264, 275, 260, 60, 112, 97, 158, + 156, 277, 135, 80, 73, 104, 106, 105, 298, 113, + 114, 280, 281, 279, 130, 65, 97, 282, 247, 12, + 206, 65, 12, 284, 285, 156, 107, 132, 108, 290, + 132, 203, 74, 15, 76, 77, 294, 199, 295, 296, + 191, 12, 183, 124, 207, 204, 297, 227, 300, 6, + 228, 226, 122, 19, 20, 33, 34, 36, 37, 35, + 38, 39, 40, 41, 21, 22, 121, 288, 212, 123, + 178, 213, 211, 262, 23, 24, 25, 26, 27, 28, + 29, 287, 193, 272, 30, 31, 32, 18, 79, 209, + 12, 177, 210, 208, 256, 257, 293, 78, 6, 303, + 16, 17, 19, 20, 33, 34, 36, 37, 35, 38, + 39, 40, 41, 21, 22, 224, 301, 221, 225, 223, + 222, 220, 292, 23, 24, 25, 26, 27, 28, 29, + 278, 134, 253, 30, 31, 32, 18, 218, 251, 12, + 219, 217, 215, 180, 161, 216, 214, 6, 179, 16, + 17, 19, 20, 33, 34, 36, 37, 35, 38, 39, + 40, 41, 21, 22, 3, 178, 255, 81, 115, 168, + 72, 67, 23, 24, 25, 26, 27, 28, 29, 177, + 165, 163, 30, 31, 32, 18, 112, 162, 154, 118, + 70, 168, 110, 72, 159, 100, 166, 103, 16, 17, + 102, 59, 116, 111, 104, 106, 105, 117, 113, 114, + 239, 83, 84, 85, 86, 87, 88, 89, 90, 91, + 92, 93, 94, 95, 96, 107, 99, 108, 11, 10, + 9, 14, 8, 268, 13, 7, 69, 1, } var exprPact = [...]int{ - 274, -1000, -39, -1000, -1000, 226, 274, -1000, -1000, -1000, - -1000, -1000, 389, 229, 51, -1000, 293, 288, 228, -1000, + 276, -1000, -40, -1000, -1000, 202, 276, -1000, -1000, -1000, + -1000, -1000, 438, 231, 259, -1000, 340, 331, 230, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 33, 33, 33, 33, 33, 33, 33, 33, - 33, 33, 33, 33, 33, 33, 33, 226, -1000, 47, - 467, 373, -1000, -1000, -1000, -1000, 177, 174, -39, 272, - 238, -1000, 46, 260, 370, 222, 212, 209, -1000, -1000, - 274, 274, -1000, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, -1000, 372, -1000, - -1000, 234, -1000, -1000, 420, -1000, 363, -1000, -1000, -1000, - -1000, 164, 357, 422, 52, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 421, -1000, 340, 337, 336, 325, 203, 254, - 194, 235, 161, 250, 322, 189, 152, 244, -25, 18, - 18, -21, -21, -66, -66, -66, -66, -55, -55, -55, - -55, -55, -55, -1000, 234, 164, 164, 164, 242, -1000, - 266, -1000, 154, -1000, 237, -1000, 264, 332, 310, 408, - 382, 380, 360, 358, -1000, -1000, -1000, -1000, -1000, -1000, - 103, 235, 160, 124, 158, 409, 131, 141, 103, 274, - 134, 236, 206, -1000, -1000, 188, -1000, 324, 137, 234, - 201, 420, 315, -1000, 339, 287, -1000, -1000, -1000, -1000, + 33, 33, 33, 33, 33, 33, 33, 202, -1000, 47, + 214, 433, -1000, -1000, -1000, -1000, 156, 147, -40, 300, + 277, -1000, 46, 257, 374, 229, 219, 203, -1000, -1000, + 276, 276, -1000, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 276, 276, 276, -1000, 432, -1000, + -1000, 113, -1000, -1000, 389, -1000, 431, -1000, 425, -1000, + -1000, -1000, 224, 424, 436, 52, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 415, -1000, 423, 409, 392, 387, 185, + 273, 161, 254, 143, 271, 325, 217, 210, 268, -26, + 17, 17, -22, -22, -67, -67, -67, -67, -56, -56, + -56, -56, -56, -56, -1000, 113, 224, 224, 224, 262, + -1000, 283, -1000, -1000, 184, -1000, 251, -1000, 282, 335, + 314, 388, 383, 363, 361, 293, -1000, -1000, -1000, -1000, + -1000, -1000, 49, 254, 137, 124, 152, 413, 132, 153, + 49, 276, 120, 249, 138, -1000, -1000, 110, -1000, 382, + 208, 113, 209, 389, 376, -1000, 414, 339, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 130, 200, - 27, 200, 282, -49, 164, -49, 122, 70, 276, 126, - 127, -1000, -1000, 113, -1000, 274, 406, -1000, -1000, 210, - -1000, -1000, -1000, -1000, -1000, -1000, 103, 27, 200, 27, - -1000, -1000, 234, -1000, -49, -1000, 138, -1000, -1000, -1000, - 36, 273, 271, 73, 103, 92, -1000, 314, -1000, 27, - -1000, 387, 28, 27, 15, -49, -49, 270, -1000, -1000, - 198, 59, 27, -1000, -1000, -49, 313, -1000, -1000, 192, - 309, 55, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 136, 196, 27, 196, 315, -49, 224, -49, 70, 71, + 324, 160, 128, -1000, -1000, 119, -1000, 276, 375, -1000, + -1000, 244, -1000, -1000, -1000, -1000, -1000, -1000, 49, 27, + 196, 27, -1000, -1000, 113, -1000, -49, -1000, 197, -1000, + -1000, -1000, 36, 322, 308, 73, 49, 107, -1000, 366, + -1000, 27, -1000, 341, 28, 27, 15, -49, -49, 287, + -1000, -1000, 239, 59, 27, -1000, -1000, -49, 360, -1000, + -1000, 206, 343, 53, -1000, } var exprPgo = [...]int{ - 0, 477, 16, 70, 2, 9, 410, 3, 15, 11, - 476, 475, 474, 473, 7, 453, 449, 448, 447, 446, - 413, 445, 13, 5, 442, 441, 440, 6, 439, 438, - 437, 4, 436, 435, 8, 434, 1, 433, 0, + 0, 487, 16, 70, 2, 9, 414, 3, 15, 11, + 486, 485, 484, 483, 7, 482, 481, 480, 479, 478, + 417, 476, 13, 5, 457, 453, 452, 6, 451, 450, + 447, 4, 446, 445, 8, 444, 1, 442, 0, } var exprR1 = [...]int{ @@ -340,16 +333,16 @@ var exprR1 = [...]int{ 15, 15, 15, 15, 15, 19, 3, 3, 3, 3, 14, 14, 14, 10, 10, 9, 9, 9, 9, 22, 22, 23, 23, 23, 23, 23, 23, 28, 28, 21, - 21, 21, 21, 33, 29, 31, 31, 32, 32, 32, - 30, 27, 27, 27, 27, 27, 27, 27, 27, 34, - 35, 35, 37, 37, 26, 26, 26, 26, 26, 26, - 26, 24, 24, 24, 24, 24, 24, 24, 25, 25, - 25, 25, 25, 25, 25, 17, 17, 17, 17, 17, + 21, 21, 21, 21, 33, 29, 31, 31, 32, 32, + 32, 30, 27, 27, 27, 27, 27, 27, 27, 27, + 34, 35, 35, 37, 37, 26, 26, 26, 26, 26, + 26, 26, 24, 24, 24, 24, 24, 24, 24, 25, + 25, 25, 25, 25, 25, 25, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 20, 20, 18, 18, 18, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 12, 12, 12, 12, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 12, 38, 5, - 5, 4, 4, 4, 4, + 17, 20, 20, 18, 18, 18, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 38, + 5, 5, 4, 4, 4, 4, } var exprR2 = [...]int{ @@ -361,84 +354,84 @@ var exprR2 = [...]int{ 5, 5, 6, 7, 7, 12, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, 3, 1, 2, 1, 2, 2, 2, 2, 2, 2, 3, 1, - 1, 2, 1, 2, 2, 3, 3, 1, 3, 3, - 2, 1, 1, 1, 3, 2, 3, 3, 3, 3, - 1, 3, 1, 1, 3, 3, 3, 3, 3, 3, + 1, 2, 1, 2, 2, 2, 3, 3, 1, 3, + 3, 2, 1, 1, 1, 3, 2, 3, 3, 3, + 3, 1, 3, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 0, 1, 1, 2, 2, 1, 1, 1, 1, 1, + 4, 0, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, - 3, 4, 4, 3, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, + 1, 3, 4, 4, 3, 3, } var exprChk = [...]int{ -1000, -1, -2, -6, -7, -14, 23, -11, -15, -17, - -18, -19, 15, -12, -16, 7, 73, 74, 61, 27, + -18, -19, 15, -12, -16, 7, 74, 75, 61, 27, 28, 38, 39, 48, 49, 50, 51, 52, 53, 54, 58, 59, 60, 29, 30, 33, 31, 32, 34, 35, - 36, 37, 64, 65, 66, 73, 74, 75, 76, 77, - 78, 67, 68, 71, 72, 69, 70, -22, -23, -28, - 44, -3, 21, 22, 14, 68, -7, -6, -2, -10, + 36, 37, 65, 66, 67, 74, 75, 76, 77, 78, + 79, 68, 69, 72, 73, 70, 71, -22, -23, -28, + 44, -3, 21, 22, 14, 69, -7, -6, -2, -10, 2, -9, 5, 23, 23, -4, 25, 26, 7, 7, 23, -20, 40, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -23, -3, -21, - -33, -27, -29, -30, 41, 43, 42, 62, -9, -37, - -25, 23, 45, 46, 5, -26, -24, 6, 24, 24, - 16, 2, 19, 16, 12, 68, 13, 14, -8, 7, - -14, 23, -7, 7, 23, 23, 23, -7, -2, -2, + -33, -27, -29, -30, 41, 43, 42, 62, 64, -9, + -37, -25, 23, 45, 46, 5, -26, -24, 6, 24, + 24, 16, 2, 19, 16, 12, 69, 13, 14, -8, + 7, -14, 23, -7, 7, 23, 23, 23, -7, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - -2, -2, -2, 6, -27, 65, 19, 64, -35, -34, - 5, 6, -27, 6, -32, -31, 5, 12, 68, 71, - 72, 69, 70, 67, -9, 6, 6, 6, 6, 2, - 24, 19, 9, -36, -22, 44, -14, -8, 24, 19, - -7, 7, -5, 24, 5, -5, 24, 19, -27, -27, - -27, 19, 12, 24, 19, 12, 8, 4, 7, 8, - 4, 7, 8, 4, 7, 8, 4, 7, 8, 4, - 7, 8, 4, 7, 8, 4, 7, -4, -8, -38, - -36, -22, 63, 9, 44, 9, -36, 47, 24, -36, - -22, 24, -4, -7, 24, 19, 19, 24, 24, 6, - -34, 6, -31, 2, 5, 6, 24, -36, -22, -36, - 8, -38, -27, -38, 9, 5, -13, 55, 56, 57, - 9, 24, 24, -36, 24, -7, 5, 19, -4, -36, - -38, 23, -38, -36, 44, 9, 9, 24, -4, 24, - 6, 5, -36, -38, -38, 9, 19, 24, -38, 6, - 19, 6, 24, + -2, -2, -2, -2, 6, -27, 66, 19, 65, -35, + -34, 5, 6, 6, -27, 6, -32, -31, 5, 12, + 69, 72, 73, 70, 71, 68, -9, 6, 6, 6, + 6, 2, 24, 19, 9, -36, -22, 44, -14, -8, + 24, 19, -7, 7, -5, 24, 5, -5, 24, 19, + -27, -27, -27, 19, 12, 24, 19, 12, 8, 4, + 7, 8, 4, 7, 8, 4, 7, 8, 4, 7, + 8, 4, 7, 8, 4, 7, 8, 4, 7, -4, + -8, -38, -36, -22, 63, 9, 44, 9, -36, 47, + 24, -36, -22, 24, -4, -7, 24, 19, 19, 24, + 24, 6, -34, 6, -31, 2, 5, 6, 24, -36, + -22, -36, 8, -38, -27, -38, 9, 5, -13, 55, + 56, 57, 9, 24, 24, -36, 24, -7, 5, 19, + -4, -36, -38, 23, -38, -36, 44, 9, 9, 24, + -4, 24, 6, 5, -36, -38, -38, 9, 19, 24, + -38, 6, 19, 6, 24, } var exprDef = [...]int{ 0, -2, 1, 2, 3, 10, 0, 4, 5, 6, - 7, 8, 0, 0, 0, 142, 0, 0, 0, 154, - 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, - 165, 166, 167, 145, 146, 147, 148, 149, 150, 151, - 152, 153, 140, 140, 140, 140, 140, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 140, 11, 69, 71, + 7, 8, 0, 0, 0, 143, 0, 0, 0, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, + 166, 167, 168, 146, 147, 148, 149, 150, 151, 152, + 153, 154, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 11, 69, 71, 0, 0, 56, 57, 58, 59, 3, 2, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 143, 144, - 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, + 0, 63, 0, 0, 0, 0, 0, 0, 144, 145, + 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 72, - 73, 74, 75, 76, 79, 80, 0, 82, 91, 92, - 93, 0, 0, 0, 0, 102, 103, 77, 9, 12, - 60, 61, 0, 62, 0, 0, 0, 0, 0, 0, - 0, 0, 3, 142, 0, 0, 0, 3, 125, 126, + 73, 74, 75, 76, 79, 80, 0, 82, 0, 92, + 93, 94, 0, 0, 0, 0, 103, 104, 77, 9, + 12, 60, 61, 0, 62, 0, 0, 0, 0, 0, + 0, 0, 0, 3, 143, 0, 0, 0, 3, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, - 137, 138, 139, 78, 95, 0, 0, 0, 83, 100, - 0, 81, 0, 84, 90, 87, 0, 0, 0, 0, - 0, 0, 0, 0, 64, 65, 66, 67, 68, 38, - 45, 0, 13, 0, 0, 0, 0, 0, 49, 0, - 3, 142, 0, 173, 169, 0, 174, 0, 96, 97, - 98, 0, 0, 94, 0, 0, 109, 116, 123, 108, - 115, 122, 104, 111, 118, 105, 112, 119, 106, 113, - 120, 107, 114, 121, 110, 117, 124, 47, 0, 14, - 17, 33, 0, 21, 0, 25, 0, 0, 0, 0, - 0, 37, 51, 3, 50, 0, 0, 171, 172, 0, - 101, 99, 88, 89, 85, 86, 46, 18, 34, 35, - 168, 22, 41, 26, 29, 39, 0, 42, 43, 44, - 15, 0, 0, 0, 52, 3, 170, 0, 48, 36, - 30, 0, 16, 19, 0, 23, 27, 0, 53, 54, - 0, 0, 20, 24, 28, 31, 0, 40, 32, 0, - 0, 0, 55, + 137, 138, 139, 140, 78, 96, 0, 0, 0, 84, + 101, 0, 81, 83, 0, 85, 91, 88, 0, 0, + 0, 0, 0, 0, 0, 0, 64, 65, 66, 67, + 68, 38, 45, 0, 13, 0, 0, 0, 0, 0, + 49, 0, 3, 143, 0, 174, 170, 0, 175, 0, + 97, 98, 99, 0, 0, 95, 0, 0, 110, 117, + 124, 109, 116, 123, 105, 112, 119, 106, 113, 120, + 107, 114, 121, 108, 115, 122, 111, 118, 125, 47, + 0, 14, 17, 33, 0, 21, 0, 25, 0, 0, + 0, 0, 0, 37, 51, 3, 50, 0, 0, 172, + 173, 0, 102, 100, 89, 90, 86, 87, 46, 18, + 34, 35, 169, 22, 41, 26, 29, 39, 0, 42, + 43, 44, 15, 0, 0, 0, 52, 3, 171, 0, + 48, 36, 30, 0, 16, 19, 0, 23, 27, 0, + 53, 54, 0, 0, 20, 24, 28, 31, 0, 40, + 32, 0, 0, 0, 55, } var exprTok1 = [...]int{ @@ -453,7 +446,7 @@ var exprTok2 = [...]int{ 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, + 72, 73, 74, 75, 76, 77, 78, 79, } var exprTok3 = [...]int{ 0, @@ -465,7 +458,6 @@ var exprErrorMessages = [...]struct { msg string }{} -//line yaccpar:1 /* parser for yacc output */ @@ -798,1032 +790,865 @@ exprdefault: case 1: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:115 { exprlex.(*parser).expr = exprDollar[1].Expr } case 2: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:118 { exprVAL.Expr = exprDollar[1].LogExpr } case 3: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:119 { exprVAL.Expr = exprDollar[1].MetricExpr } case 4: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:123 { exprVAL.MetricExpr = exprDollar[1].RangeAggregationExpr } case 5: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:124 { exprVAL.MetricExpr = exprDollar[1].VectorAggregationExpr } case 6: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:125 { exprVAL.MetricExpr = exprDollar[1].BinOpExpr } case 7: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:126 { exprVAL.MetricExpr = exprDollar[1].LiteralExpr } case 8: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:127 { exprVAL.MetricExpr = exprDollar[1].LabelReplaceExpr } case 9: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:128 { exprVAL.MetricExpr = exprDollar[2].MetricExpr } case 10: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:132 { exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) } case 11: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:133 { exprVAL.LogExpr = newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].PipelineExpr) } case 12: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:134 { exprVAL.LogExpr = exprDollar[2].LogExpr } case 13: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:138 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].duration, nil, nil) } case 14: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:139 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].duration, nil, exprDollar[3].OffsetExpr) } case 15: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:140 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[4].duration, nil, nil) } case 16: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:141 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[4].duration, nil, exprDollar[5].OffsetExpr) } case 17: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:142 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].duration, exprDollar[3].UnwrapExpr, nil) } case 18: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:143 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].duration, exprDollar[4].UnwrapExpr, exprDollar[3].OffsetExpr) } case 19: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:144 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[4].duration, exprDollar[5].UnwrapExpr, nil) } case 20: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:145 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[4].duration, exprDollar[6].UnwrapExpr, exprDollar[5].OffsetExpr) } case 21: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:146 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[3].duration, exprDollar[2].UnwrapExpr, nil) } case 22: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:147 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[1].Selector), exprDollar[3].duration, exprDollar[2].UnwrapExpr, exprDollar[4].OffsetExpr) } case 23: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:148 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[5].duration, exprDollar[3].UnwrapExpr, nil) } case 24: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:149 { exprVAL.LogRangeExpr = newLogRange(newMatcherExpr(exprDollar[2].Selector), exprDollar[5].duration, exprDollar[3].UnwrapExpr, exprDollar[6].OffsetExpr) } case 25: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:150 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].PipelineExpr), exprDollar[3].duration, nil, nil) } case 26: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:151 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].PipelineExpr), exprDollar[3].duration, nil, exprDollar[4].OffsetExpr) } case 27: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:152 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[2].Selector), exprDollar[3].PipelineExpr), exprDollar[5].duration, nil, nil) } case 28: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:153 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[2].Selector), exprDollar[3].PipelineExpr), exprDollar[5].duration, nil, exprDollar[6].OffsetExpr) } case 29: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:154 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].PipelineExpr), exprDollar[4].duration, exprDollar[3].UnwrapExpr, nil) } case 30: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:155 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[2].PipelineExpr), exprDollar[4].duration, exprDollar[3].UnwrapExpr, exprDollar[5].OffsetExpr) } case 31: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:156 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[2].Selector), exprDollar[3].PipelineExpr), exprDollar[6].duration, exprDollar[4].UnwrapExpr, nil) } case 32: exprDollar = exprS[exprpt-7 : exprpt+1] -//line pkg/logql/expr.y:157 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[2].Selector), exprDollar[3].PipelineExpr), exprDollar[6].duration, exprDollar[4].UnwrapExpr, exprDollar[7].OffsetExpr) } case 33: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:158 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[3].PipelineExpr), exprDollar[2].duration, nil, nil) } case 34: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:159 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[4].PipelineExpr), exprDollar[2].duration, nil, exprDollar[3].OffsetExpr) } case 35: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:160 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[3].PipelineExpr), exprDollar[2].duration, exprDollar[4].UnwrapExpr, nil) } case 36: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:161 { exprVAL.LogRangeExpr = newLogRange(newPipelineExpr(newMatcherExpr(exprDollar[1].Selector), exprDollar[4].PipelineExpr), exprDollar[2].duration, exprDollar[5].UnwrapExpr, exprDollar[3].OffsetExpr) } case 37: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:162 { exprVAL.LogRangeExpr = exprDollar[2].LogRangeExpr } case 39: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:167 { exprVAL.UnwrapExpr = newUnwrapExpr(exprDollar[3].str, "") } case 40: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:168 { exprVAL.UnwrapExpr = newUnwrapExpr(exprDollar[5].str, exprDollar[3].ConvOp) } case 41: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:169 { exprVAL.UnwrapExpr = exprDollar[1].UnwrapExpr.addPostFilter(exprDollar[3].LabelFilter) } case 42: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:173 { exprVAL.ConvOp = OpConvBytes } case 43: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:174 { exprVAL.ConvOp = OpConvDuration } case 44: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:175 { exprVAL.ConvOp = OpConvDurationSeconds } case 45: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:179 { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp, nil, nil) } case 46: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:180 { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[5].LogRangeExpr, exprDollar[1].RangeOp, nil, &exprDollar[3].str) } case 47: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:181 { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp, exprDollar[5].Grouping, nil) } case 48: exprDollar = exprS[exprpt-7 : exprpt+1] -//line pkg/logql/expr.y:182 { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[5].LogRangeExpr, exprDollar[1].RangeOp, exprDollar[7].Grouping, &exprDollar[3].str) } case 49: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:187 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].MetricExpr, exprDollar[1].VectorOp, nil, nil) } case 50: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:188 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].MetricExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } case 51: exprDollar = exprS[exprpt-5 : exprpt+1] -//line pkg/logql/expr.y:189 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].MetricExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } case 52: exprDollar = exprS[exprpt-6 : exprpt+1] -//line pkg/logql/expr.y:191 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].MetricExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } case 53: exprDollar = exprS[exprpt-7 : exprpt+1] -//line pkg/logql/expr.y:192 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].MetricExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } case 54: exprDollar = exprS[exprpt-7 : exprpt+1] -//line pkg/logql/expr.y:193 { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[6].MetricExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, &exprDollar[4].str) } case 55: exprDollar = exprS[exprpt-12 : exprpt+1] -//line pkg/logql/expr.y:198 { exprVAL.LabelReplaceExpr = mustNewLabelReplaceExpr(exprDollar[3].MetricExpr, exprDollar[5].str, exprDollar[7].str, exprDollar[9].str, exprDollar[11].str) } case 56: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:202 { exprVAL.Filter = labels.MatchRegexp } case 57: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:203 { exprVAL.Filter = labels.MatchEqual } case 58: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:204 { exprVAL.Filter = labels.MatchNotRegexp } case 59: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:205 { exprVAL.Filter = labels.MatchNotEqual } case 60: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:209 { exprVAL.Selector = exprDollar[2].Matchers } case 61: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:210 { exprVAL.Selector = exprDollar[2].Matchers } case 62: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:211 { } case 63: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:215 { exprVAL.Matchers = []*labels.Matcher{exprDollar[1].Matcher} } case 64: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:216 { exprVAL.Matchers = append(exprDollar[1].Matchers, exprDollar[3].Matcher) } case 65: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:220 { exprVAL.Matcher = mustNewMatcher(labels.MatchEqual, exprDollar[1].str, exprDollar[3].str) } case 66: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:221 { exprVAL.Matcher = mustNewMatcher(labels.MatchNotEqual, exprDollar[1].str, exprDollar[3].str) } case 67: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:222 { exprVAL.Matcher = mustNewMatcher(labels.MatchRegexp, exprDollar[1].str, exprDollar[3].str) } case 68: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:223 { exprVAL.Matcher = mustNewMatcher(labels.MatchNotRegexp, exprDollar[1].str, exprDollar[3].str) } case 69: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:227 { exprVAL.PipelineExpr = MultiStageExpr{exprDollar[1].PipelineStage} } case 70: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:228 { exprVAL.PipelineExpr = append(exprDollar[1].PipelineExpr, exprDollar[2].PipelineStage) } case 71: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:232 { exprVAL.PipelineStage = exprDollar[1].LineFilters } case 72: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:233 { exprVAL.PipelineStage = exprDollar[2].LabelParser } case 73: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:234 { exprVAL.PipelineStage = exprDollar[2].JSONExpressionParser } case 74: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:235 { exprVAL.PipelineStage = &labelFilterExpr{LabelFilterer: exprDollar[2].LabelFilter} } case 75: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:236 { exprVAL.PipelineStage = exprDollar[2].LineFormatExpr } case 76: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:237 { exprVAL.PipelineStage = exprDollar[2].LabelFormatExpr } case 77: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:241 { exprVAL.LineFilters = newLineFilterExpr(nil, exprDollar[1].Filter, exprDollar[2].str) } case 78: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:242 { exprVAL.LineFilters = newLineFilterExpr(exprDollar[1].LineFilters, exprDollar[2].Filter, exprDollar[3].str) } case 79: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:245 { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeJSON, "") } case 80: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:246 { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeLogfmt, "") } case 81: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:247 { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeRegexp, exprDollar[2].str) } case 82: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:248 { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeUnpack, "") } case 83: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:252 { - exprVAL.JSONExpressionParser = newJSONExpressionParser(exprDollar[2].JSONExpressionList) + exprVAL.LabelParser = newLabelParserExpr(OpParserTypePattern, exprDollar[2].str) } case 84: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:254 { - exprVAL.LineFormatExpr = newLineFmtExpr(exprDollar[2].str) + exprVAL.JSONExpressionParser = newJSONExpressionParser(exprDollar[2].JSONExpressionList) } case 85: + exprDollar = exprS[exprpt-2 : exprpt+1] + { + exprVAL.LineFormatExpr = newLineFmtExpr(exprDollar[2].str) + } + case 86: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:257 { exprVAL.LabelFormat = log.NewRenameLabelFmt(exprDollar[1].str, exprDollar[3].str) } - case 86: + case 87: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:258 { exprVAL.LabelFormat = log.NewTemplateLabelFmt(exprDollar[1].str, exprDollar[3].str) } - case 87: + case 88: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:262 { exprVAL.LabelsFormat = []log.LabelFmt{exprDollar[1].LabelFormat} } - case 88: + case 89: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:263 { exprVAL.LabelsFormat = append(exprDollar[1].LabelsFormat, exprDollar[3].LabelFormat) } - case 90: + case 91: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:267 { exprVAL.LabelFormatExpr = newLabelFmtExpr(exprDollar[2].LabelsFormat) } - case 91: + case 92: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:270 { exprVAL.LabelFilter = log.NewStringLabelFilter(exprDollar[1].Matcher) } - case 92: + case 93: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:271 { exprVAL.LabelFilter = exprDollar[1].UnitFilter } - case 93: + case 94: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:272 { exprVAL.LabelFilter = exprDollar[1].NumberFilter } - case 94: + case 95: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:273 { exprVAL.LabelFilter = exprDollar[2].LabelFilter } - case 95: + case 96: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:274 { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[2].LabelFilter) } - case 96: + case 97: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:275 { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 97: + case 98: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:276 { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 98: + case 99: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:277 { exprVAL.LabelFilter = log.NewOrLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 99: + case 100: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:281 { exprVAL.JSONExpression = log.NewJSONExpr(exprDollar[1].str, exprDollar[3].str) } - case 100: + case 101: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:284 { exprVAL.JSONExpressionList = []log.JSONExpression{exprDollar[1].JSONExpression} } - case 101: + case 102: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:285 { exprVAL.JSONExpressionList = append(exprDollar[1].JSONExpressionList, exprDollar[3].JSONExpression) } - case 102: + case 103: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:289 { exprVAL.UnitFilter = exprDollar[1].DurationFilter } - case 103: + case 104: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:290 { exprVAL.UnitFilter = exprDollar[1].BytesFilter } - case 104: - exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:293 - { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].duration) - } case 105: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:294 { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].duration) } case 106: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:295 { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } case 107: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:296 { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].duration) } case 108: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:297 { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } case 109: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:298 { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].duration) } case 110: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:299 { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } case 111: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:303 { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } case 112: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:304 { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].bytes) } case 113: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:305 { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } case 114: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:306 { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].bytes) } case 115: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:307 { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } case 116: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:308 { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].bytes) } case 117: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:309 { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } case 118: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:313 { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } case 119: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:314 { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 120: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:315 { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 121: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:316 { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 122: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:317 { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 123: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:318 { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 124: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:319 { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 125: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + } + case 126: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:325 { exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 126: + case 127: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:326 { exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 127: + case 128: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:327 { exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 128: + case 129: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:328 { exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 129: + case 130: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:329 { exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 130: + case 131: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:330 { exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 131: + case 132: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:331 { exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 132: + case 133: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:332 { exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 133: + case 134: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:333 { exprVAL.BinOpExpr = mustNewBinOpExpr("^", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 134: + case 135: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:334 { exprVAL.BinOpExpr = mustNewBinOpExpr("==", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 135: + case 136: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:335 { exprVAL.BinOpExpr = mustNewBinOpExpr("!=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 136: + case 137: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:336 { exprVAL.BinOpExpr = mustNewBinOpExpr(">", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 137: + case 138: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:337 { exprVAL.BinOpExpr = mustNewBinOpExpr(">=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 138: + case 139: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:338 { exprVAL.BinOpExpr = mustNewBinOpExpr("<", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 139: + case 140: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:339 { exprVAL.BinOpExpr = mustNewBinOpExpr("<=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 140: + case 141: exprDollar = exprS[exprpt-0 : exprpt+1] -//line pkg/logql/expr.y:343 { exprVAL.BinOpModifier = BinOpOptions{} } - case 141: + case 142: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:344 { exprVAL.BinOpModifier = BinOpOptions{ReturnBool: true} } - case 142: + case 143: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:348 { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str, false) } - case 143: + case 144: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:349 { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, false) } - case 144: + case 145: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:350 { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, true) } - case 145: + case 146: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:354 { exprVAL.VectorOp = OpTypeSum } - case 146: + case 147: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:355 { exprVAL.VectorOp = OpTypeAvg } - case 147: + case 148: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:356 { exprVAL.VectorOp = OpTypeCount } - case 148: + case 149: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:357 { exprVAL.VectorOp = OpTypeMax } - case 149: + case 150: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:358 { exprVAL.VectorOp = OpTypeMin } - case 150: + case 151: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:359 { exprVAL.VectorOp = OpTypeStddev } - case 151: + case 152: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:360 { exprVAL.VectorOp = OpTypeStdvar } - case 152: + case 153: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:361 { exprVAL.VectorOp = OpTypeBottomK } - case 153: + case 154: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:362 { exprVAL.VectorOp = OpTypeTopK } - case 154: + case 155: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:366 { exprVAL.RangeOp = OpRangeTypeCount } - case 155: + case 156: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:367 { exprVAL.RangeOp = OpRangeTypeRate } - case 156: + case 157: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:368 { exprVAL.RangeOp = OpRangeTypeBytes } - case 157: + case 158: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:369 { exprVAL.RangeOp = OpRangeTypeBytesRate } - case 158: + case 159: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:370 { exprVAL.RangeOp = OpRangeTypeAvg } - case 159: + case 160: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:371 { exprVAL.RangeOp = OpRangeTypeSum } - case 160: + case 161: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:372 { exprVAL.RangeOp = OpRangeTypeMin } - case 161: + case 162: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:373 { exprVAL.RangeOp = OpRangeTypeMax } - case 162: + case 163: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:374 { exprVAL.RangeOp = OpRangeTypeStdvar } - case 163: + case 164: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:375 { exprVAL.RangeOp = OpRangeTypeStddev } - case 164: + case 165: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:376 { exprVAL.RangeOp = OpRangeTypeQuantile } - case 165: + case 166: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:377 { exprVAL.RangeOp = OpRangeTypeFirst } - case 166: + case 167: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:378 { exprVAL.RangeOp = OpRangeTypeLast } - case 167: + case 168: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:379 { exprVAL.RangeOp = OpRangeTypeAbsent } - case 168: + case 169: exprDollar = exprS[exprpt-2 : exprpt+1] -//line pkg/logql/expr.y:383 { exprVAL.OffsetExpr = newOffsetExpr(exprDollar[2].duration) } - case 169: + case 170: exprDollar = exprS[exprpt-1 : exprpt+1] -//line pkg/logql/expr.y:386 { exprVAL.Labels = []string{exprDollar[1].str} } - case 170: + case 171: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:387 { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 171: + case 172: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:391 { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 172: + case 173: exprDollar = exprS[exprpt-4 : exprpt+1] -//line pkg/logql/expr.y:392 { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} } - case 173: + case 174: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:393 { exprVAL.Grouping = &grouping{without: false, groups: nil} } - case 174: + case 175: exprDollar = exprS[exprpt-3 : exprpt+1] -//line pkg/logql/expr.y:394 { exprVAL.Grouping = &grouping{without: true, groups: nil} } diff --git a/pkg/logql/functions_test.go b/pkg/logql/functions_test.go index 31af9b9ec82e1..128829aa48409 100644 --- a/pkg/logql/functions_test.go +++ b/pkg/logql/functions_test.go @@ -17,6 +17,7 @@ func Test_Extractor(t *testing.T) { `sum(count_over_time({job="mysql"}[5m]))`, `sum(count_over_time({job="mysql"} | json [5m]))`, `sum(count_over_time({job="mysql"} | logfmt [5m]))`, + `sum(count_over_time({job="mysql"} | pattern " bar " [5m]))`, `sum(count_over_time({job="mysql"} | regexp "(?Pfoo|bar)" [5m]))`, `sum(count_over_time({job="mysql"} | regexp "(?Pfoo|bar)" [5m] offset 1h))`, `topk(10,sum(rate({region="us-east1"}[5m])) by (name))`, diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index ebb0291eca179..85ba038034538 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -55,10 +55,11 @@ var tokens = map[string]int{ OpTypeLTE: LTE, // parsers - OpParserTypeJSON: JSON, - OpParserTypeRegexp: REGEXP, - OpParserTypeLogfmt: LOGFMT, - OpParserTypeUnpack: UNPACK, + OpParserTypeJSON: JSON, + OpParserTypeRegexp: REGEXP, + OpParserTypeLogfmt: LOGFMT, + OpParserTypeUnpack: UNPACK, + OpParserTypePattern: PATTERN, // fmt OpFmtLabel: LABEL_FMT, diff --git a/pkg/logql/log/parser.go b/pkg/logql/log/parser.go index 85642afa533f7..421ac34de0407 100644 --- a/pkg/logql/log/parser.go +++ b/pkg/logql/log/parser.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/loki/pkg/logql/log/jsonexpr" "github.com/grafana/loki/pkg/logql/log/logfmt" + "github.com/grafana/loki/pkg/logql/log/pattern" "github.com/grafana/loki/pkg/logqlmodel" jsoniter "github.com/json-iterator/go" @@ -212,14 +213,6 @@ func NewRegexpParser(re string) (*RegexpParser, error) { }, nil } -func mustNewRegexParser(re string) *RegexpParser { - r, err := NewRegexpParser(re) - if err != nil { - panic(err) - } - return r -} - func (r *RegexpParser) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { for i, value := range r.regex.FindSubmatch(line) { if name, ok := r.nameIndex[i]; ok { @@ -265,6 +258,49 @@ func (l *LogfmtParser) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { func (l *LogfmtParser) RequiredLabelNames() []string { return []string{} } +type PatternParser struct { + matcher pattern.Matcher + names []string +} + +func NewPatternParser(pn string) (*PatternParser, error) { + m, err := pattern.New(pn) + if err != nil { + return nil, err + } + for _, name := range m.Names() { + if !model.LabelName(name).IsValid() { + return nil, fmt.Errorf("invalid capture label name '%s'", name) + } + } + return &PatternParser{ + matcher: m, + names: m.Names(), + }, nil +} + +func (l *PatternParser) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { + if lbs.ParserLabelHints().NoLabels() { + return line, true + } + matches := l.matcher.Matches(line) + names := l.names[:len(matches)] + for i, m := range matches { + name := names[i] + if !lbs.parserKeyHints.ShouldExtract(name) { + continue + } + if lbs.BaseHas(name) { + name = name + duplicateSuffix + } + + lbs.Set(name, string(m)) + } + return line, true +} + +func (l *PatternParser) RequiredLabelNames() []string { return []string{} } + type JSONExpressionParser struct { expressions map[string][]interface{} } diff --git a/pkg/logql/log/parser_test.go b/pkg/logql/log/parser_test.go index cf192af0a2a31..74fc215e5b443 100644 --- a/pkg/logql/log/parser_test.go +++ b/pkg/logql/log/parser_test.go @@ -408,8 +408,9 @@ func Benchmark_Parser(b *testing.B) { {"json", jsonLine, NewJSONParser(), []string{"response_latency_seconds"}}, {"unpack", packedLike, NewUnpackParser(), []string{"pod"}}, {"logfmt", logfmtLine, NewLogfmtParser(), []string{"info", "throughput", "org_id"}}, - {"regex greedy", nginxline, mustNewRegexParser(`GET (?P.*?)/\?`), []string{"path"}}, - {"regex status digits", nginxline, mustNewRegexParser(`HTTP/1.1" (?P\d{3}) `), []string{"statuscode"}}, + {"regex greedy", nginxline, mustStage(NewRegexpParser(`GET (?P.*?)/\?`)), []string{"path"}}, + {"regex status digits", nginxline, mustStage(NewRegexpParser(`HTTP/1.1" (?P\d{3}) `)), []string{"statuscode"}}, + {"pattern", nginxline, mustStage(NewPatternParser(`<_> " <_>"<_>`)), []string{"path"}}, } { b.Run(tt.name, func(b *testing.B) { line := []byte(tt.line) @@ -433,6 +434,13 @@ func Benchmark_Parser(b *testing.B) { } } +func mustStage(s Stage, err error) Stage { + if err != nil { + panic(err) + } + return s +} + func TestNewRegexpParser(t *testing.T) { tests := []struct { name string @@ -460,14 +468,14 @@ func TestNewRegexpParser(t *testing.T) { func Test_regexpParser_Parse(t *testing.T) { tests := []struct { name string - parser *RegexpParser + parser Stage line []byte lbs labels.Labels want labels.Labels }{ { "no matches", - mustNewRegexParser("(?Pfoo|bar)buzz"), + mustStage(NewRegexpParser("(?Pfoo|bar)buzz")), []byte("blah"), labels.Labels{ {Name: "app", Value: "foo"}, @@ -478,7 +486,7 @@ func Test_regexpParser_Parse(t *testing.T) { }, { "double matches", - mustNewRegexParser("(?P.*)buzz"), + mustStage(NewRegexpParser("(?P.*)buzz")), []byte("matchebuzz barbuzz"), labels.Labels{ {Name: "app", Value: "bar"}, @@ -490,7 +498,7 @@ func Test_regexpParser_Parse(t *testing.T) { }, { "duplicate labels", - mustNewRegexParser("(?Pbar)buzz"), + mustStage(NewRegexpParser("(?Pbar)buzz")), []byte("barbuzz"), labels.Labels{ {Name: "bar", Value: "foo"}, @@ -502,7 +510,7 @@ func Test_regexpParser_Parse(t *testing.T) { }, { "multiple labels extracted", - mustNewRegexParser("status=(?P\\w+),latency=(?P\\w+)(ms|ns)"), + mustStage(NewRegexpParser("status=(?P\\w+),latency=(?P\\w+)(ms|ns)")), []byte("status=200,latency=500ms"), labels.Labels{ {Name: "app", Value: "foo"}, @@ -733,3 +741,68 @@ func Test_unpackParser_Parse(t *testing.T) { }) } } + +func Test_PatternParser(t *testing.T) { + tests := []struct { + pattern string + line []byte + lbs labels.Labels + want labels.Labels + }{ + { + ` [<_>] " <_>" `, + []byte(`127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326`), + labels.Labels{ + {Name: "foo", Value: "bar"}, + }, + labels.Labels{ + {Name: "foo", Value: "bar"}, + {Name: "ip", Value: "127.0.0.1"}, + {Name: "userid", Value: "user-identifier"}, + {Name: "user", Value: "frank"}, + {Name: "method", Value: "GET"}, + {Name: "path", Value: "/apache_pb.gif"}, + {Name: "status", Value: "200"}, + {Name: "size", Value: "2326"}, + }, + }, + { + `<_> msg=" () "`, + []byte(`level=debug ts=2021-05-19T07:54:26.864644382Z caller=logging.go:66 traceID=7fbb92fd0eb9c65d msg="POST /loki/api/v1/push (204) 1.238734ms"`), + labels.Labels{ + {Name: "method", Value: "bar"}, + }, + labels.Labels{ + {Name: "method", Value: "bar"}, + {Name: "method_extracted", Value: "POST"}, + {Name: "path", Value: "/loki/api/v1/push"}, + {Name: "status", Value: "204"}, + {Name: "duration", Value: "1.238734ms"}, + }, + }, + { + `foo "`, + []byte(`bar`), + labels.Labels{ + {Name: "method", Value: "bar"}, + }, + labels.Labels{ + {Name: "method", Value: "bar"}, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.pattern, func(t *testing.T) { + t.Parallel() + b := NewBaseLabelsBuilder().ForLabels(tt.lbs, tt.lbs.Hash()) + b.Reset() + pp, err := NewPatternParser(tt.pattern) + require.NoError(t, err) + _, _ = pp.Process(tt.line, b) + sort.Sort(tt.want) + require.Equal(t, tt.want, b.Labels()) + }) + } +} diff --git a/pkg/logql/log/pattern/ast.go b/pkg/logql/log/pattern/ast.go new file mode 100644 index 0000000000000..f8309d1c480b1 --- /dev/null +++ b/pkg/logql/log/pattern/ast.go @@ -0,0 +1,83 @@ +package pattern + +import ( + "fmt" + "unicode/utf8" +) + +type node interface { + fmt.Stringer +} + +type expr []node + +func (e expr) hasCapture() bool { + return e.captureCount() != 0 +} + +func (e expr) validate() error { + if !e.hasCapture() { + return ErrNoCapture + } + // if there is at least 2 node, verify that none are consecutive. + if len(e) >= 2 { + for i := 0; i < len(e); i = i + 2 { + if i+1 >= len(e) { + break + } + if _, ok := e[i].(capture); ok { + if _, ok := e[i+1].(capture); ok { + return fmt.Errorf("found consecutive capture: %w", ErrInvalidExpr) + } + } + } + } + caps := e.captures() + uniq := map[string]struct{}{} + for _, c := range caps { + if _, ok := uniq[c]; ok { + return fmt.Errorf("duplicate capture name (%s): %w", c, ErrInvalidExpr) + } + uniq[c] = struct{}{} + } + return nil +} + +func (e expr) captures() (captures []string) { + for _, n := range e { + if c, ok := n.(capture); ok && !c.isUnamed() { + captures = append(captures, c.String()) + } + } + return +} + +func (e expr) captureCount() (count int) { + return len(e.captures()) +} + +type capture string + +func (c capture) String() string { + return string(c) +} + +func (c capture) isUnamed() bool { + return string(c) == underscore +} + +type literals []byte + +func (l literals) String() string { + return string(l) +} + +func runesToLiterals(rs []rune) literals { + res := make([]byte, len(rs)*utf8.UTFMax) + count := 0 + for _, r := range rs { + count += utf8.EncodeRune(res[count:], r) + } + res = res[:count] + return res +} diff --git a/pkg/logql/log/pattern/expr.y b/pkg/logql/log/pattern/expr.y new file mode 100644 index 0000000000000..24729a3c08564 --- /dev/null +++ b/pkg/logql/log/pattern/expr.y @@ -0,0 +1,45 @@ +%{ + +package pattern + +%} + +%union{ + Expr []node + Node node + + literal rune + Literals []rune + str string + token int +} + +%start root + +%type expr +%type node +%type literals + +%token IDENTIFIER +%token LITERAL +%token LESS_THAN MORE_THAN UNDERSCORE + +%% + +root: + expr { exprlex.(*lexer).expr = $1 }; + +expr: + node { $$ = []node{$1} } + | expr node { $$ = append($1, $2) } + ; + +node: + IDENTIFIER { $$ = capture($1) } + | literals { $$ = runesToLiterals($1) } + ; + +literals: + LITERAL { $$ = []rune{$1} } + | literals LITERAL { $$ = append($1, $2) } +%% diff --git a/pkg/logql/log/pattern/expr.y.go b/pkg/logql/log/pattern/expr.y.go new file mode 100644 index 0000000000000..70332638544c5 --- /dev/null +++ b/pkg/logql/log/pattern/expr.y.go @@ -0,0 +1,466 @@ +// Code generated by goyacc -p expr -o pkg/logql/log/pattern/expr.y.go pkg/logql/log/pattern/expr.y. DO NOT EDIT. + + +package pattern + +import __yyfmt__ "fmt" + + +type exprSymType struct { + yys int + Expr []node + Node node + + literal rune + Literals []rune + str string + token int +} + +const IDENTIFIER = 57346 +const LITERAL = 57347 +const LESS_THAN = 57348 +const MORE_THAN = 57349 +const UNDERSCORE = 57350 + +var exprToknames = [...]string{ + "$end", + "error", + "$unk", + "IDENTIFIER", + "LITERAL", + "LESS_THAN", + "MORE_THAN", + "UNDERSCORE", +} +var exprStatenames = [...]string{} + +const exprEofCode = 1 +const exprErrCode = 2 +const exprInitialStackSize = 16 + + +var exprExca = [...]int{ + -1, 1, + 1, -1, + -2, 0, +} + +const exprPrivate = 57344 + +const exprLast = 8 + +var exprAct = [...]int{ + + 4, 6, 8, 3, 5, 2, 7, 1, +} +var exprPact = [...]int{ + + -4, -1000, -4, -1000, -1000, -3, -1000, -1000, -1000, +} +var exprPgo = [...]int{ + + 0, 7, 5, 3, 4, +} +var exprR1 = [...]int{ + + 0, 1, 2, 2, 3, 3, 4, 4, +} +var exprR2 = [...]int{ + + 0, 1, 1, 2, 1, 1, 1, 2, +} +var exprChk = [...]int{ + + -1000, -1, -2, -3, 4, -4, 5, -3, 5, +} +var exprDef = [...]int{ + + 0, -2, 1, 2, 4, 5, 6, 3, 7, +} +var exprTok1 = [...]int{ + + 1, +} +var exprTok2 = [...]int{ + + 2, 3, 4, 5, 6, 7, 8, +} +var exprTok3 = [...]int{ + 0, +} + +var exprErrorMessages = [...]struct { + state int + token int + msg string +}{} + + +/* parser for yacc output */ + +var ( + exprDebug = 0 + exprErrorVerbose = false +) + +type exprLexer interface { + Lex(lval *exprSymType) int + Error(s string) +} + +type exprParser interface { + Parse(exprLexer) int + Lookahead() int +} + +type exprParserImpl struct { + lval exprSymType + stack [exprInitialStackSize]exprSymType + char int +} + +func (p *exprParserImpl) Lookahead() int { + return p.char +} + +func exprNewParser() exprParser { + return &exprParserImpl{} +} + +const exprFlag = -1000 + +func exprTokname(c int) string { + if c >= 1 && c-1 < len(exprToknames) { + if exprToknames[c-1] != "" { + return exprToknames[c-1] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func exprStatname(s int) string { + if s >= 0 && s < len(exprStatenames) { + if exprStatenames[s] != "" { + return exprStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func exprErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !exprErrorVerbose { + return "syntax error" + } + + for _, e := range exprErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + exprTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := exprPact[state] + for tok := TOKSTART; tok-1 < len(exprToknames); tok++ { + if n := base + tok; n >= 0 && n < exprLast && exprChk[exprAct[n]] == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if exprDef[state] == -2 { + i := 0 + for exprExca[i] != -1 || exprExca[i+1] != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; exprExca[i] >= 0; i += 2 { + tok := exprExca[i] + if tok < TOKSTART || exprExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if exprExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += exprTokname(tok) + } + return res +} + +func exprlex1(lex exprLexer, lval *exprSymType) (char, token int) { + token = 0 + char = lex.Lex(lval) + if char <= 0 { + token = exprTok1[0] + goto out + } + if char < len(exprTok1) { + token = exprTok1[char] + goto out + } + if char >= exprPrivate { + if char < exprPrivate+len(exprTok2) { + token = exprTok2[char-exprPrivate] + goto out + } + } + for i := 0; i < len(exprTok3); i += 2 { + token = exprTok3[i+0] + if token == char { + token = exprTok3[i+1] + goto out + } + } + +out: + if token == 0 { + token = exprTok2[1] /* unknown char */ + } + if exprDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", exprTokname(token), uint(char)) + } + return char, token +} + +func exprParse(exprlex exprLexer) int { + return exprNewParser().Parse(exprlex) +} + +func (exprrcvr *exprParserImpl) Parse(exprlex exprLexer) int { + var exprn int + var exprVAL exprSymType + var exprDollar []exprSymType + _ = exprDollar // silence set and not used + exprS := exprrcvr.stack[:] + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + exprstate := 0 + exprrcvr.char = -1 + exprtoken := -1 // exprrcvr.char translated into internal numbering + defer func() { + // Make sure we report no lookahead when not parsing. + exprstate = -1 + exprrcvr.char = -1 + exprtoken = -1 + }() + exprp := -1 + goto exprstack + +ret0: + return 0 + +ret1: + return 1 + +exprstack: + /* put a state and value onto the stack */ + if exprDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", exprTokname(exprtoken), exprStatname(exprstate)) + } + + exprp++ + if exprp >= len(exprS) { + nyys := make([]exprSymType, len(exprS)*2) + copy(nyys, exprS) + exprS = nyys + } + exprS[exprp] = exprVAL + exprS[exprp].yys = exprstate + +exprnewstate: + exprn = exprPact[exprstate] + if exprn <= exprFlag { + goto exprdefault /* simple state */ + } + if exprrcvr.char < 0 { + exprrcvr.char, exprtoken = exprlex1(exprlex, &exprrcvr.lval) + } + exprn += exprtoken + if exprn < 0 || exprn >= exprLast { + goto exprdefault + } + exprn = exprAct[exprn] + if exprChk[exprn] == exprtoken { /* valid shift */ + exprrcvr.char = -1 + exprtoken = -1 + exprVAL = exprrcvr.lval + exprstate = exprn + if Errflag > 0 { + Errflag-- + } + goto exprstack + } + +exprdefault: + /* default state action */ + exprn = exprDef[exprstate] + if exprn == -2 { + if exprrcvr.char < 0 { + exprrcvr.char, exprtoken = exprlex1(exprlex, &exprrcvr.lval) + } + + /* look through exception table */ + xi := 0 + for { + if exprExca[xi+0] == -1 && exprExca[xi+1] == exprstate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + exprn = exprExca[xi+0] + if exprn < 0 || exprn == exprtoken { + break + } + } + exprn = exprExca[xi+1] + if exprn < 0 { + goto ret0 + } + } + if exprn == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + exprlex.Error(exprErrorMessage(exprstate, exprtoken)) + Nerrs++ + if exprDebug >= 1 { + __yyfmt__.Printf("%s", exprStatname(exprstate)) + __yyfmt__.Printf(" saw %s\n", exprTokname(exprtoken)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for exprp >= 0 { + exprn = exprPact[exprS[exprp].yys] + exprErrCode + if exprn >= 0 && exprn < exprLast { + exprstate = exprAct[exprn] /* simulate a shift of "error" */ + if exprChk[exprstate] == exprErrCode { + goto exprstack + } + } + + /* the current p has no shift on "error", pop stack */ + if exprDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", exprS[exprp].yys) + } + exprp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if exprDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", exprTokname(exprtoken)) + } + if exprtoken == exprEofCode { + goto ret1 + } + exprrcvr.char = -1 + exprtoken = -1 + goto exprnewstate /* try again in the same state */ + } + } + + /* reduction by production exprn */ + if exprDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", exprn, exprStatname(exprstate)) + } + + exprnt := exprn + exprpt := exprp + _ = exprpt // guard against "declared and not used" + + exprp -= exprR2[exprn] + // exprp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if exprp+1 >= len(exprS) { + nyys := make([]exprSymType, len(exprS)*2) + copy(nyys, exprS) + exprS = nyys + } + exprVAL = exprS[exprp+1] + + /* consult goto table to find next state */ + exprn = exprR1[exprn] + exprg := exprPgo[exprn] + exprj := exprg + exprS[exprp].yys + 1 + + if exprj >= exprLast { + exprstate = exprAct[exprg] + } else { + exprstate = exprAct[exprj] + if exprChk[exprstate] != -exprn { + exprstate = exprAct[exprg] + } + } + // dummy call; replaced with literal code + switch exprnt { + + case 1: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprlex.(*lexer).expr = exprDollar[1].Expr + } + case 2: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.Expr = []node{exprDollar[1].Node} + } + case 3: + exprDollar = exprS[exprpt-2 : exprpt+1] + { + exprVAL.Expr = append(exprDollar[1].Expr, exprDollar[2].Node) + } + case 4: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.Node = capture(exprDollar[1].str) + } + case 5: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.Node = runesToLiterals(exprDollar[1].Literals) + } + case 6: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.Literals = []rune{exprDollar[1].literal} + } + case 7: + exprDollar = exprS[exprpt-2 : exprpt+1] + { + exprVAL.Literals = append(exprDollar[1].Literals, exprDollar[2].literal) + } + } + goto exprstack /* stack new state and value */ +} diff --git a/pkg/logql/log/pattern/lexer.go b/pkg/logql/log/pattern/lexer.go new file mode 100644 index 0000000000000..2f867f93dc305 --- /dev/null +++ b/pkg/logql/log/pattern/lexer.go @@ -0,0 +1,62 @@ +package pattern + +type lexer struct { + data []byte + p, pe, cs int + ts, te, act int + + lastnewline int + curline int + + errs []parseError + expr []node +} + +func newLexer() *lexer { + lex := &lexer{} + lex.init() + return lex +} + +func (lex *lexer) setData(data []byte) { + lex.data = data + lex.pe = len(data) + lex.lastnewline = -1 + lex.curline = 1 +} + +// Error implements exprLexer interface generated by yacc (yyLexer) +func (lex *lexer) Error(e string) { + lex.errs = append(lex.errs, newParseError(e, lex.curline, lex.curcol())) +} + +// curcol calculates the current token's start column based on the last newline position +// returns a 1-indexed value +func (lex *lexer) curcol() int { + return (lex.ts + 1 /* 1-indexed columns */) - (lex.lastnewline + 1 /* next after newline */) +} + +func (lex *lexer) handle(token int, err error) int { + if err != nil { + lex.Error(err.Error()) + return LEXER_ERROR + } + return token +} + +func (lex *lexer) token() string { + return string(lex.data[lex.ts:lex.te]) +} + +// nolint +func (lex *lexer) identifier(out *exprSymType) (int, error) { + t := lex.token() + out.str = t[1 : len(t)-1] + return IDENTIFIER, nil +} + +// nolint +func (lex *lexer) literal(out *exprSymType) (int, error) { + out.literal = rune(lex.data[lex.ts]) + return LITERAL, nil +} diff --git a/pkg/logql/log/pattern/lexer.rl b/pkg/logql/log/pattern/lexer.rl new file mode 100644 index 0000000000000..7b1d25467c45a --- /dev/null +++ b/pkg/logql/log/pattern/lexer.rl @@ -0,0 +1,43 @@ +package pattern + +%%{ + machine pattern; + write data; + access lex.; + variable p lex.p; + variable pe lex.pe; + prepush { + if len(lex.stack) <= lex.top { + lex.stack = append(lex.stack, 0) + } + } +}%% + +const LEXER_ERROR = 0 + +%%{ + identifier = '<' (alpha| '_') (alnum | '_' )* '>'; + literal = any; +}%% + +func (lex *lexer) Lex(out *exprSymType) int { + eof := lex.pe + tok := 0 + + %%{ + + main := |* + identifier => { tok = lex.handle(lex.identifier(out)); fbreak; }; + literal => { tok = lex.handle(lex.literal(out)); fbreak; }; + *|; + + write exec; + }%% + + return tok; +} + + +func (lex *lexer) init() { + %% write init; +} diff --git a/pkg/logql/log/pattern/lexer.rl.go b/pkg/logql/log/pattern/lexer.rl.go new file mode 100644 index 0000000000000..4f5a80c43f448 --- /dev/null +++ b/pkg/logql/log/pattern/lexer.rl.go @@ -0,0 +1,241 @@ + +//line pkg/logql/log/pattern/lexer.rl:1 +package pattern + + +//line pkg/logql/log/pattern/lexer.rl.go:7 +var _pattern_actions []byte = []byte{ + 0, 1, 0, 1, 1, 1, 2, 1, 3, + 1, 4, 1, 5, 1, 6, +} + +var _pattern_key_offsets []byte = []byte{ + 0, 8, 9, +} + +var _pattern_trans_keys []byte = []byte{ + 62, 95, 48, 57, 65, 90, 97, 122, + 60, 95, 65, 90, 97, 122, +} + +var _pattern_single_lengths []byte = []byte{ + 2, 1, 1, +} + +var _pattern_range_lengths []byte = []byte{ + 3, 0, 2, +} + +var _pattern_index_offsets []byte = []byte{ + 0, 6, 8, +} + +var _pattern_trans_targs []byte = []byte{ + 1, 0, 0, 0, 0, 1, 2, 1, + 0, 0, 0, 1, 1, 1, +} + +var _pattern_trans_actions []byte = []byte{ + 7, 0, 0, 0, 0, 13, 5, 9, + 0, 0, 0, 11, 13, 11, +} + +var _pattern_to_state_actions []byte = []byte{ + 0, 1, 0, +} + +var _pattern_from_state_actions []byte = []byte{ + 0, 3, 0, +} + +var _pattern_eof_trans []byte = []byte{ + 13, 0, 14, +} + +const pattern_start int = 1 +const pattern_first_final int = 1 +const pattern_error int = -1 + +const pattern_en_main int = 1 + + +//line pkg/logql/log/pattern/lexer.rl:14 + + +const LEXER_ERROR = 0 + + +//line pkg/logql/log/pattern/lexer.rl:21 + + +func (lex *lexer) Lex(out *exprSymType) int { + eof := lex.pe + tok := 0 + + +//line pkg/logql/log/pattern/lexer.rl.go:77 + { + var _klen int + var _trans int + var _acts int + var _nacts uint + var _keys int + if ( lex.p) == ( lex.pe) { + goto _test_eof + } +_resume: + _acts = int(_pattern_from_state_actions[ lex.cs]) + _nacts = uint(_pattern_actions[_acts]); _acts++ + for ; _nacts > 0; _nacts-- { + _acts++ + switch _pattern_actions[_acts - 1] { + case 1: +//line NONE:1 + lex.ts = ( lex.p) + +//line pkg/logql/log/pattern/lexer.rl.go:97 + } + } + + _keys = int(_pattern_key_offsets[ lex.cs]) + _trans = int(_pattern_index_offsets[ lex.cs]) + + _klen = int(_pattern_single_lengths[ lex.cs]) + if _klen > 0 { + _lower := int(_keys) + var _mid int + _upper := int(_keys + _klen - 1) + for { + if _upper < _lower { + break + } + + _mid = _lower + ((_upper - _lower) >> 1) + switch { + case lex.data[( lex.p)] < _pattern_trans_keys[_mid]: + _upper = _mid - 1 + case lex.data[( lex.p)] > _pattern_trans_keys[_mid]: + _lower = _mid + 1 + default: + _trans += int(_mid - int(_keys)) + goto _match + } + } + _keys += _klen + _trans += _klen + } + + _klen = int(_pattern_range_lengths[ lex.cs]) + if _klen > 0 { + _lower := int(_keys) + var _mid int + _upper := int(_keys + (_klen << 1) - 2) + for { + if _upper < _lower { + break + } + + _mid = _lower + (((_upper - _lower) >> 1) & ^1) + switch { + case lex.data[( lex.p)] < _pattern_trans_keys[_mid]: + _upper = _mid - 2 + case lex.data[( lex.p)] > _pattern_trans_keys[_mid + 1]: + _lower = _mid + 2 + default: + _trans += int((_mid - int(_keys)) >> 1) + goto _match + } + } + _trans += _klen + } + +_match: +_eof_trans: + lex.cs = int(_pattern_trans_targs[_trans]) + + if _pattern_trans_actions[_trans] == 0 { + goto _again + } + + _acts = int(_pattern_trans_actions[_trans]) + _nacts = uint(_pattern_actions[_acts]); _acts++ + for ; _nacts > 0; _nacts-- { + _acts++ + switch _pattern_actions[_acts-1] { + case 2: +//line NONE:1 + lex.te = ( lex.p)+1 + + case 3: +//line pkg/logql/log/pattern/lexer.rl:30 + lex.te = ( lex.p)+1 +{ tok = lex.handle(lex.identifier(out)); ( lex.p)++; goto _out + } + case 4: +//line pkg/logql/log/pattern/lexer.rl:31 + lex.te = ( lex.p)+1 +{ tok = lex.handle(lex.literal(out)); ( lex.p)++; goto _out + } + case 5: +//line pkg/logql/log/pattern/lexer.rl:31 + lex.te = ( lex.p) +( lex.p)-- +{ tok = lex.handle(lex.literal(out)); ( lex.p)++; goto _out + } + case 6: +//line pkg/logql/log/pattern/lexer.rl:31 +( lex.p) = ( lex.te) - 1 +{ tok = lex.handle(lex.literal(out)); ( lex.p)++; goto _out + } +//line pkg/logql/log/pattern/lexer.rl.go:191 + } + } + +_again: + _acts = int(_pattern_to_state_actions[ lex.cs]) + _nacts = uint(_pattern_actions[_acts]); _acts++ + for ; _nacts > 0; _nacts-- { + _acts++ + switch _pattern_actions[_acts-1] { + case 0: +//line NONE:1 + lex.ts = 0 + +//line pkg/logql/log/pattern/lexer.rl.go:205 + } + } + + ( lex.p)++ + if ( lex.p) != ( lex.pe) { + goto _resume + } + _test_eof: {} + if ( lex.p) == eof { + if _pattern_eof_trans[ lex.cs] > 0 { + _trans = int(_pattern_eof_trans[ lex.cs] - 1) + goto _eof_trans + } + } + + _out: {} + } + +//line pkg/logql/log/pattern/lexer.rl:35 + + + return tok; +} + + +func (lex *lexer) init() { + +//line pkg/logql/log/pattern/lexer.rl.go:233 + { + lex.cs = pattern_start + lex.ts = 0 + lex.te = 0 + lex.act = 0 + } + +//line pkg/logql/log/pattern/lexer.rl:43 +} diff --git a/pkg/logql/log/pattern/lexer_test.go b/pkg/logql/log/pattern/lexer_test.go new file mode 100644 index 0000000000000..ff4d61591fb39 --- /dev/null +++ b/pkg/logql/log/pattern/lexer_test.go @@ -0,0 +1,47 @@ +package pattern + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Lex(t *testing.T) { + for _, tc := range []struct { + input string + expected []int + }{ + {`_foo`, []int{LITERAL, LITERAL, LITERAL, LITERAL}}, + {``, []int{LITERAL}}, + {`<_1foo>`, []int{IDENTIFIER}}, + {`<_1foo> bar `, []int{IDENTIFIER, LITERAL, LITERAL, LITERAL, LITERAL, LITERAL, IDENTIFIER}}, + {`<1foo>`, []int{LITERAL, LITERAL, LITERAL, LITERAL, LITERAL, LITERAL}}, + } { + tc := tc + t.Run(tc.input, func(t *testing.T) { + actual := []int{} + l := newLexer() + l.setData([]byte(tc.input)) + for { + var lval exprSymType + tok := l.Lex(&lval) + if tok == 0 { + break + } + actual = append(actual, tok) + } + assert.Equal(t, toksToStrings(tc.expected), toksToStrings(actual)) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func toksToStrings(toks []int) []string { + strings := make([]string, len(toks)) + for i, tok := range toks { + strings[i] = exprToknames[tok-exprPrivate+1] + } + return strings +} diff --git a/pkg/logql/log/pattern/parser.go b/pkg/logql/log/pattern/parser.go new file mode 100644 index 0000000000000..d1bc2515c9cb2 --- /dev/null +++ b/pkg/logql/log/pattern/parser.go @@ -0,0 +1,50 @@ +package pattern + +import "fmt" + +const underscore = "_" + +var tokens = map[int]string{ + LESS_THAN: "<", + MORE_THAN: ">", + UNDERSCORE: underscore, +} + +func init() { + // Improve the error messages coming out of yacc. + exprErrorVerbose = true + for tok, str := range tokens { + exprToknames[tok-exprPrivate+1] = str + } +} + +func parseExpr(input string) (expr, error) { + l := newLexer() + l.setData([]byte(input)) + e := exprNewParser().Parse(l) + if e != 0 || len(l.errs) > 0 { + return nil, l.errs[0] + } + return l.expr, nil +} + +// parseError is what is returned when we failed to parse. +type parseError struct { + msg string + line, col int +} + +func (p parseError) Error() string { + if p.col == 0 && p.line == 0 { + return p.msg + } + return fmt.Sprintf("parse error at line %d, col %d: %s", p.line, p.col, p.msg) +} + +func newParseError(msg string, line, col int) parseError { + return parseError{ + msg: msg, + line: line, + col: col, + } +} diff --git a/pkg/logql/log/pattern/parser_test.go b/pkg/logql/log/pattern/parser_test.go new file mode 100644 index 0000000000000..b46f8a2027ea0 --- /dev/null +++ b/pkg/logql/log/pattern/parser_test.go @@ -0,0 +1,59 @@ +package pattern + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_Parse(t *testing.T) { + for _, tc := range []struct { + input string + expected expr + err error + }{ + { + " bar f ", + expr{capture("foo"), literals(" bar f "), capture("f")}, + nil, + }, + { + "", + expr{literals(""), capture("bar")}, + nil, + }, + { + "<>", + expr{literals("<>")}, + nil, + }, + { + "<_>", + expr{capture("_")}, + nil, + }, + { + "<1_>", + expr{literals("<1_>")}, + nil, + }, + { + ` - [<_>] " <_>" `, + expr{capture("ip"), literals(" - "), capture("user"), literals(" ["), capture("_"), literals(`] "`), capture("method"), literals(" "), capture("path"), literals(" "), capture('_'), literals(`" `), capture("status"), literals(" "), capture("size"), literals(" "), capture("url"), literals(" "), capture("user_agent")}, + nil, + }, + } { + tc := tc + actual, err := parseExpr(tc.input) + if tc.err != nil || err != nil { + require.Equal(t, tc.err, err) + return + } + require.Equal(t, tc.expected, actual) + } +} diff --git a/pkg/logql/log/pattern/pattern.go b/pkg/logql/log/pattern/pattern.go new file mode 100644 index 0000000000000..c1c65739fb965 --- /dev/null +++ b/pkg/logql/log/pattern/pattern.go @@ -0,0 +1,95 @@ +package pattern + +import ( + "bytes" + "errors" +) + +var ( + ErrNoCapture = errors.New("at least one capture is required") + ErrInvalidExpr = errors.New("invalid expression") +) + +type Matcher interface { + Matches(in []byte) [][]byte + Names() []string +} + +type matcher struct { + e expr + + captures [][]byte + names []string +} + +func New(in string) (Matcher, error) { + e, err := parseExpr(in) + if err != nil { + return nil, err + } + if err := e.validate(); err != nil { + return nil, err + } + return &matcher{ + e: e, + captures: make([][]byte, 0, e.captureCount()), + names: e.captures(), + }, nil +} + +// Matches matches the given line with the provided pattern. +// Matches invalidates the previous returned captures array. +func (m *matcher) Matches(in []byte) [][]byte { + if len(in) == 0 { + return nil + } + if len(m.e) == 0 { + return nil + } + captures := m.captures[:0] + expr := m.e + if ls, ok := expr[0].(literals); ok { + i := bytes.Index(in, ls) + if i != 0 { + return nil + } + in = in[len(ls):] + expr = expr[1:] + } + if len(expr) == 0 { + return nil + } + // from now we have capture - literals - capture ... (literals)? + for len(expr) != 0 { + if len(expr) == 1 { // we're ending on a capture. + if !(expr[0].(capture)).isUnamed() { + captures = append(captures, in) + } + return captures + } + cap := expr[0].(capture) + ls := expr[1].(literals) + expr = expr[2:] + i := bytes.Index(in, ls) + if i == -1 { + // if a capture is missed we return up to the end as the capture. + if !cap.isUnamed() { + captures = append(captures, in) + } + return captures + } + + if cap.isUnamed() { + in = in[len(ls)+i:] + continue + } + captures = append(captures, in[:i]) + in = in[len(ls)+i:] + } + + return captures +} + +func (m *matcher) Names() []string { + return m.names +} diff --git a/pkg/logql/log/pattern/pattern_test.go b/pkg/logql/log/pattern/pattern_test.go new file mode 100644 index 0000000000000..6a0a760dd29fa --- /dev/null +++ b/pkg/logql/log/pattern/pattern_test.go @@ -0,0 +1,162 @@ +package pattern + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +var fixtures = []struct { + expr string + in string + expected []string +}{ + { + "foo bar", + "foo buzz bar", + []string{"buzz"}, + }, + { + "foo bar", + "foo buzz bar", + []string{"buzz", ""}, + }, + { + " bar", + " bar", + []string{"", ""}, + }, + { + "?<_>", + `/api/plugins/versioncheck?slugIn=snuids-trafficlights-panel,input,gel&grafanaVersion=7.0.0-beta1`, + []string{"/api/plugins/versioncheck"}, + }, + { + "?<_>", + `/api/plugins/status`, + []string{"/api/plugins/status"}, + }, + { + // Common Log Format + ` [<_>] " <_>" `, + `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326`, + []string{"127.0.0.1", "user-identifier", "frank", "GET", "/apache_pb.gif", "200", "2326"}, + }, + { + // Combined Log Format + ` - - [<_>] " <_>" `, + `35.191.8.106 - - [19/May/2021:07:21:49 +0000] "GET /api/plugins/versioncheck?slugIn=snuids-trafficlights-panel,input,gel&grafanaVersion=7.0.0-beta1 HTTP/1.1" 200 107 "-" "Go-http-client/2.0" "80.153.74.144, 34.120.177.193" "TLSv1.3" "DE" "DEBW"`, + []string{"35.191.8.106", "GET", "/api/plugins/versioncheck?slugIn=snuids-trafficlights-panel,input,gel&grafanaVersion=7.0.0-beta1", "200", "107"}, + }, + { + // MySQL + `<_> [] [] [] `, + `2020-08-06T14:25:02.835618Z 0 [Note] [MY-012487] [InnoDB] DDL log recovery : begin`, + []string{"0", "Note", "MY-012487", "InnoDB"}, + }, + { + // MySQL + `<_> [] `, + `2021-05-19T07:40:12.215792Z 42761518 [Note] Aborted connection 42761518 to db: 'hosted_grafana' user: 'hosted_grafana' host: '10.36.4.122' (Got an error reading communication packets)`, + []string{"42761518", "Note"}, + }, + { + // Kubernetes api-server + ` <_> <_> ] `, + `W0519 07:46:47.647050 1 clientconn.go:1223] grpc: addrConn.createTransport failed to connect to {https://kubernetes-etcd-1.kubernetes-etcd:2379 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp 10.32.85.85:2379: connect: connection refused". Reconnecting...`, + []string{"W0519", "clientconn.go:1223"}, + }, + { + // Cassandra + ` []<_> in .<_>`, + `INFO [Service Thread] 2021-05-19 07:40:12,130 GCInspector.java:284 - ParNew GC in 248ms. CMS Old Gen: 5043436640 -> 5091062064; Par Eden Space: 671088640 -> 0; Par Survivor Space: 70188280 -> 60139760`, + []string{"INFO", "Service Thread", "248ms"}, + }, + { + // Cortex & Loki distributor + `<_> msg=" () "`, + `level=debug ts=2021-05-19T07:54:26.864644382Z caller=logging.go:66 traceID=7fbb92fd0eb9c65d msg="POST /loki/api/v1/push (204) 1.238734ms"`, + []string{"POST", "/loki/api/v1/push", "204", "1.238734ms"}, + }, + { + // Etcd + `<_> <_> | : <_> peer <_> tcp :<_>`, + `2021-05-19 08:16:50.181436 W | rafthttp: health check for peer fd8275e521cfb532 could not connect: dial tcp 10.32.85.85:2380: connect: connection refused`, + []string{"W", "rafthttp", "fd8275e521cfb532", "10.32.85.85"}, + }, + { + // Kafka + `<_>] [Log partition=, dir=] `, + `[2021-05-19 08:35:28,681] INFO [Log partition=p-636-L-fs-117, dir=/data/kafka-logs] Deleting segment 455976081 (kafka.log.Log)`, + []string{"INFO", "p-636-L-fs-117", "/data/kafka-logs"}, + }, + { + // Elastic + `<_>][][] [] []`, + `[2021-05-19T06:54:06,994][INFO ][o.e.c.m.MetaDataMappingService] [1f605d47-8454-4bfb-a67f-49f318bf837a] [usage-stats-2021.05.19/O2Je9IbmR8CqFyUvNpTttA] update_mapping [report]`, + []string{"INFO ", "o.e.c.m.MetaDataMappingService", "1f605d47-8454-4bfb-a67f-49f318bf837a", "usage-stats-2021.05.19/O2Je9IbmR8CqFyUvNpTttA"}, + }, + { + // Envoy + `<_> " <_>" <_> "" "" <_> <_> ""`, + `[2016-04-15T20:17:00.310Z] "POST /api/v1/locations HTTP/2" 204 - 154 0 226 100 "10.0.35.28" "nsq2http" "cc21d9b0-cf5c-432b-8c7e-98aeb7988cd2" "locations" "tcp://10.0.2.1:80"`, + []string{"POST", "/api/v1/locations", "204", "154", "0", "226", "100", "10.0.35.28", "nsq2http", "tcp://10.0.2.1:80"}, + }, +} + +func Test_matcher_Matches(t *testing.T) { + for _, tt := range fixtures { + tt := tt + t.Run(tt.expr, func(t *testing.T) { + t.Parallel() + m, err := New(tt.expr) + require.NoError(t, err) + actual := m.Matches([]byte(tt.in)) + var actualStrings []string + for _, a := range actual { + actualStrings = append(actualStrings, string(a)) + } + require.Equal(t, tt.expected, actualStrings) + }) + } +} + +var res [][]byte + +func Benchmark_matcher_Matches(b *testing.B) { + for _, tt := range fixtures { + b.Run(tt.expr, func(b *testing.B) { + b.ReportAllocs() + m, err := New(tt.expr) + require.NoError(b, err) + b.ResetTimer() + l := []byte(tt.in) + for n := 0; n < b.N; n++ { + res = m.Matches(l) + } + }) + } +} + +func Test_Error(t *testing.T) { + for _, tt := range []struct { + name string + err error + }{ + {"", nil}, + {" ", nil}, + {"", newParseError("syntax error: unexpected $end, expecting IDENTIFIER or LITERAL", 1, 1)}, + {"<_>", ErrNoCapture}, + {"foo <_> bar <_>", ErrNoCapture}, + {"foo bar buzz", ErrNoCapture}, + {"", fmt.Errorf("found consecutive capture: %w", ErrInvalidExpr)}, + {" f", fmt.Errorf("found consecutive capture: %w", ErrInvalidExpr)}, + {" f", fmt.Errorf("duplicate capture name (f): %w", ErrInvalidExpr)}, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := New(tt.name) + require.Equal(t, tt.err, err) + }) + } +} diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 3fdc4ec65bf74..8bec4aef21146 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -1061,6 +1061,25 @@ func TestParse(t *testing.T) { }, }, }, + { + in: `{app="foo"} |= "bar" | pattern " bar " | (duration > 1s or status!= 200) and method!="POST"`, + exp: &pipelineExpr{ + left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), + pipeline: MultiStageExpr{ + newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLabelParserExpr(OpParserTypePattern, " bar "), + &labelFilterExpr{ + LabelFilterer: log.NewAndLabelFilter( + log.NewOrLabelFilter( + log.NewDurationLabelFilter(log.LabelFilterGreaterThan, "duration", 1*time.Second), + log.NewNumericLabelFilter(log.LabelFilterNotEqual, "status", 200.0), + ), + log.NewStringLabelFilter(mustNewMatcher(labels.MatchNotEqual, "method", "POST")), + ), + }, + }, + }, + }, { in: `{app="foo"} |= "bar" | json | ( status_code < 500 and status_code > 200) or latency >= 250ms `, exp: &pipelineExpr{ @@ -2508,7 +2527,6 @@ func TestParseLogSelectorExpr_equalityMatcher(t *testing.T) { } func Test_match(t *testing.T) { - tests := []struct { name string input []string @@ -2554,7 +2572,6 @@ func Test_match(t *testing.T) { } else { require.Equal(t, tt.want, got) } - }) } }