Merge remote-tracking branch 'upstream/master' into label-replace

Signed-off-by: Cyril Tovena <[email protected]>

Makefile

@@ -272,8 +272,8 @@ clean:
 	rm -rf .cache
 	rm -rf cmd/docker-driver/rootfs
 	rm -rf dist/
-	rm -rf cmd/fluent-bit/out_loki.h
-	rm -rf cmd/fluent-bit/out_loki.so
+	rm -rf cmd/fluent-bit/out_grafana_loki.h
+	rm -rf cmd/fluent-bit/out_grafana_loki.so
 	go clean $(MOD_FLAG) ./...
 
 #########
@@ -418,7 +418,7 @@ docker-driver-clean:
 # fluent-bit plugin #
 #####################
 fluent-bit-plugin:
-	go build $(DYN_GO_FLAGS) -buildmode=c-shared -o cmd/fluent-bit/out_loki.so ./cmd/fluent-bit/
+	go build $(DYN_GO_FLAGS) -buildmode=c-shared -o cmd/fluent-bit/out_grafana_loki.so ./cmd/fluent-bit/
 
 fluent-bit-image:
 	$(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) -f cmd/fluent-bit/Dockerfile .

cmd/fluent-bit/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false fluent-bit-plugin
 
 FROM fluent/fluent-bit:1.4
-COPY --from=build /src/loki/cmd/fluent-bit/out_loki.so /fluent-bit/bin
+COPY --from=build /src/loki/cmd/fluent-bit/out_grafana_loki.so /fluent-bit/bin
 COPY cmd/fluent-bit/fluent-bit.conf /fluent-bit/etc/fluent-bit.conf
 EXPOSE 2020
-CMD ["/fluent-bit/bin/fluent-bit", "-e","/fluent-bit/bin/out_loki.so", "-c", "/fluent-bit/etc/fluent-bit.conf"]
+CMD ["/fluent-bit/bin/fluent-bit", "-e","/fluent-bit/bin/out_grafana_loki.so", "-c", "/fluent-bit/etc/fluent-bit.conf"]

cmd/fluent-bit/README.md

@@ -13,7 +13,7 @@ Prerequisites:
 * Go 1.11+
 * gcc (for cgo)
 
-To build the output plugin library file (`out_loki.so`), you can use:
+To build the output plugin library file (`out_grafana_loki.so`), you can use:
 
 ```bash
 make fluent-bit-plugin

cmd/fluent-bit/fluent-bit.conf

@@ -2,7 +2,7 @@
     Name        tail
     Path        ${LOG_PATH}
 [Output]
-    Name loki
+    Name grafana-loki
     Match *
     Url ${LOKI_URL}
     BatchWait 1s

cmd/fluent-bit/out_loki.go → cmd/fluent-bit/out_grafana_loki.go

@@ -37,7 +37,7 @@ func (c *pluginConfig) Get(key string) string {
 
 //export FLBPluginRegister
 func FLBPluginRegister(ctx unsafe.Pointer) int {
-	return output.FLBPluginRegister(ctx, "loki", "Ship fluent-bit logs to Grafana Loki")
+	return output.FLBPluginRegister(ctx, "grafana-loki", "Ship fluent-bit logs to Grafana Loki")
 }
 
 //export FLBPluginInit

docs/sources/clients/aws/ecs/_index.md

@@ -154,7 +154,7 @@ All `options` of the `logConfiguration` will be automatically translated into [f
 
 ```conf
 [OUTPUT]
-    Name loki
+    Name grafana-loki
     Match awsfirelens*
     Url https://<userid>:<grafancloud apikey>@logs-prod-us-central1.grafana.net/loki/api/v1/push
     Labels {job="firelens"}

docs/sources/clients/docker-driver/configuration.md

@@ -30,7 +30,7 @@ docker run --log-driver=loki \
 ```
 
 > **Note**: The Loki logging driver still uses the json-log driver in combination with sending logs to Loki, this is mainly useful to keep the `docker logs` command working.
-> You can adjust file size and rotation using the respective log option `max-size` and `max-file`.
+> You can adjust file size and rotation using the respective log option `max-size` and `max-file`. Keep in mind that default values for these options are not taken from json-log configuration.
 > You can deactivate this behavior by setting the log option `no-file` to true.
 
 ## Change the default logging driver

docs/sources/clients/fluentbit/_index.md

@@ -53,14 +53,14 @@ First, you need to follow the [instructions](https://github.com/grafana/loki/blo
 The assuming you have Fluent Bit installed in your `$PATH` you can run the plugin using:
 
 ```bash
-fluent-bit -e /path/to/built/out_loki.so -c fluent-bit.conf
+fluent-bit -e /path/to/built/out_grafana_loki.so -c fluent-bit.conf
 ```
 
 You can also adapt your plugins.conf, removing the need to change the command line options:
 
 ```conf
 [PLUGINS]
-    Path /path/to/built/out_loki.so
+    Path /path/to/built/out_grafana_loki.so
 ```
 
 ## Configuration Options
@@ -155,7 +155,7 @@ Loki output plugin has buffering mechanism based on [`dque`](https://github.com/
 
 ```properties
 [Output]
-    Name loki
+    Name grafana-loki
     Match *
     Url http://localhost:3100/loki/api/v1/push
     Buffer true
@@ -170,7 +170,7 @@ To configure the Loki output plugin add this section to fluent-bit.conf
 
 ```properties
 [Output]
-    Name loki
+    Name grafana-loki
     Match *
     Url http://localhost:3100/loki/api/v1/push
     BatchWait 1s
@@ -184,7 +184,7 @@ To configure the Loki output plugin add this section to fluent-bit.conf
 
 ```properties
 [Output]
-    Name loki
+    Name grafana-loki
     Match *
     Url http://localhost:3100/loki/api/v1/push
     BatchWait 1s

docs/sources/configuration/_index.md

@@ -66,6 +66,8 @@ non-list parameters the value is set to the specified default.
 
 ### Use environment variables in the configuration
 
+> **Note:** This feature is only available in Loki 2.1+.
+
 You can use environment variable references in the configuration file to set values that need to be configurable during deployment.
 To do this, use:
 

docs/sources/logql/_index.md

@@ -122,7 +122,7 @@ The matching is case-sensitive by default and can be switched to case-insensitiv
 
 While line filter expressions could be placed anywhere in a pipeline, it is almost always better to have them at the beginning. This ways it will improve the performance of the query doing further processing only when a line matches.
 
-For example, while the result will be the same, the following query `{job="mysql"} |= "error" | json | line_format "{{.err}}"` will always run faster than  `{job="mysql"} | json | line_format "{{.message}}"` |= "error"`. Line filter expressions are the fastest way to filter logs after log stream selectors.
+For example, while the result will be the same, the following query `{job="mysql"} |= "error" | json | line_format "{{.err}}"` will always run faster than  `{job="mysql"} | json | line_format "{{.message}}" |= "error"`. Line filter expressions are the fastest way to filter logs after log stream selectors.
 
 #### Parser Expression
 
@@ -293,7 +293,7 @@ For example the following expression:
 
 Will extract and rewrite the log line to only contains the query and the duration of a request.
 
-You can use double quoted string for the template or single backtick \``\{{.label_name}}`\` to avoid the need to escape special characters.
+You can use double quoted string for the template or backticks `` `{{.label_name}}` `` to avoid the need to escape special characters.
 
 See [template functions](template_functions/) to learn about available functions in the template format.
 
@@ -423,6 +423,7 @@ We currently support the functions:
 
 Supported function for operating over unwrapped ranges are:
 
+- `rate(log-range)`: calculates per second rate of all values in the specified interval.
 - `sum_over_time(unwrapped-range)`: the sum of all values in the specified interval.
 - `avg_over_time(unwrapped-range)`: the average value of all points in the specified interval.
 - `max_over_time(unwrapped-range)`: the maximum value of all points in the specified interval.
@@ -431,7 +432,7 @@ Supported function for operating over unwrapped ranges are:
 - `stddev_over_time(unwrapped-range)`: the population standard deviation of the values in the specified interval.
 - `quantile_over_time(scalar,unwrapped-range)`: the φ-quantile (0 ≤ φ ≤ 1) of the values in the specified interval.
 
-Except for `sum_over_time`, `min_over_time` and `max_over_time` unwrapped range aggregations support grouping.
+Except for `sum_over_time` and `rate` unwrapped range aggregations support grouping.
 
 ```logql
 <aggr-op>([parameter,] <unwrapped-range>) [without|by (<label list>)]

pkg/logql/ast.go

@@ -604,14 +604,14 @@ func (e *rangeAggregationExpr) Selector() LogSelectorExpr {
 func (e rangeAggregationExpr) validate() error {
 	if e.grouping != nil {
 		switch e.operation {
-		case OpRangeTypeAvg, OpRangeTypeStddev, OpRangeTypeStdvar, OpRangeTypeQuantile:
+		case OpRangeTypeAvg, OpRangeTypeStddev, OpRangeTypeStdvar, OpRangeTypeQuantile, OpRangeTypeMax, OpRangeTypeMin:
 		default:
 			return fmt.Errorf("grouping not allowed for %s aggregation", e.operation)
 		}
 	}
 	if e.left.unwrap != nil {
 		switch e.operation {
-		case OpRangeTypeAvg, OpRangeTypeSum, OpRangeTypeMax, OpRangeTypeMin, OpRangeTypeStddev, OpRangeTypeStdvar, OpRangeTypeQuantile:
+		case OpRangeTypeRate, OpRangeTypeAvg, OpRangeTypeSum, OpRangeTypeMax, OpRangeTypeMin, OpRangeTypeStddev, OpRangeTypeStdvar, OpRangeTypeQuantile:
 			return nil
 		default:
 			return fmt.Errorf("invalid aggregation %s with unwrap", e.operation)
@@ -715,7 +715,10 @@ func (e *vectorAggregationExpr) Extractor() (log.SampleExtractor, error) {
 	// inject in the range vector extractor the outer groups to improve performance.
 	// This is only possible if the operation is a sum. Anything else needs all labels.
 	if r, ok := e.left.(*rangeAggregationExpr); ok && canInjectVectorGrouping(e.operation, r.operation) {
-		return r.extractor(e.grouping, len(e.grouping.groups) == 0)
+		// if the range vec operation has no grouping we can push down the vec one.
+		if r.grouping == nil {
+			return r.extractor(e.grouping)
+		}
 	}
 	return e.left.Extractor()
 }

pkg/logql/engine_test.go

@@ -85,6 +85,17 @@ func TestEngine_LogsInstantQuery(t *testing.T) {
 			},
 			promql.Vector{promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0.5}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}},
 		},
+		{
+			`rate({app="foo"} | unwrap foo [30s])`, time.Unix(60, 0), logproto.FORWARD, 10,
+			[][]logproto.Series{
+				// 30s range the lower bound of the range is not inclusive only 15 samples will make it 60 included
+				{newSeries(testSize, offset(46, constantValue(2)), `{app="foo"}`)},
+			},
+			[]SelectSampleParams{
+				{&logproto.SampleQueryRequest{Start: time.Unix(30, 0), End: time.Unix(60, 0), Selector: `rate({app="foo"} | unwrap foo[30s])`}},
+			},
+			promql.Vector{promql.Sample{Point: promql.Point{T: 60 * 1000, V: 1.0}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}},
+		},
 		{
 			`count_over_time({app="foo"} |~".+bar" [1m])`, time.Unix(60, 0), logproto.BACKWARD, 10,
 			[][]logproto.Series{
@@ -903,6 +914,25 @@ func TestEngine_RangeQuery(t *testing.T) {
 				},
 			},
 		},
+		{
+			`rate(({app=~"foo|bar"} |~".+bar" | unwrap bar)[1m])`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, 0, logproto.FORWARD, 100,
+			[][]logproto.Series{
+				{newSeries(testSize, factor(10, constantValue(2)), `{app="foo"}`), newSeries(testSize, factor(5, constantValue(2)), `{app="bar"}`)},
+			},
+			[]SelectSampleParams{
+				{&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(180, 0), Selector: `rate({app=~"foo|bar"}|~".+bar"|unwrap bar[1m])`}},
+			},
+			promql.Matrix{
+				promql.Series{
+					Metric: labels.Labels{{Name: "app", Value: "bar"}},
+					Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}},
+				},
+				promql.Series{
+					Metric: labels.Labels{{Name: "app", Value: "foo"}},
+					Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}},
+				},
+			},
+		},
 		{
 			`topk(2,rate(({app=~"foo|bar"} |~".+bar")[1m]))`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, 0, logproto.FORWARD, 100,
 			[][]logproto.Series{
@@ -2034,7 +2064,24 @@ func constant(t int64) generator {
 			Sample: logproto.Sample{
 				Timestamp: time.Unix(t, 0).UnixNano(),
 				Hash:      uint64(i),
-				Value:     1.,
+				Value:     1.0,
+			},
+		}
+	}
+}
+
+// nolint
+func constantValue(t int64) generator {
+	return func(i int64) logData {
+		return logData{
+			Entry: logproto.Entry{
+				Timestamp: time.Unix(i, 0),
+				Line:      fmt.Sprintf("%d", i),
+			},
+			Sample: logproto.Sample{
+				Timestamp: time.Unix(i, 0).UnixNano(),
+				Hash:      uint64(i),
+				Value:     float64(t),
 			},
 		}
 	}

pkg/logql/expr.y

@@ -351,5 +351,7 @@ labels:
 grouping:
       BY OPEN_PARENTHESIS labels CLOSE_PARENTHESIS        { $$ = &grouping{ without: false , groups: $3 } }
     | WITHOUT OPEN_PARENTHESIS labels CLOSE_PARENTHESIS   { $$ = &grouping{ without: true , groups: $3 } }
+    | BY OPEN_PARENTHESIS CLOSE_PARENTHESIS               { $$ = &grouping{ without: false , groups: nil } }
+    | WITHOUT OPEN_PARENTHESIS CLOSE_PARENTHESIS          { $$ = &grouping{ without: true , groups: nil } }
     ;
 %%