[release-2.9.x] Revert "LogQL: Introduce distinct (#8662)" (#10358)

Backport c030217 from #10356

---

**What this PR does / why we need it**:
This reverts commit 38b298c.

We are removing this PR #8662 because [known issue tracked
here](#9594) and some issues we
facing internally with query splitting and sharding work(hard to
parallelize, merging two `distinct` sets can have duplicates, hard to
make it work with metric queries, etc).

We decided best to remove it before make it to public release.

**Which issue(s) this PR fixes**:
Fixes #<issue number>

**Special notes for your reviewer**:

**Checklist**
- [x] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- [x] Documentation added
- [x] Tests updated
- [x] `CHANGELOG.md` updated
- [ ] If the change is worth mentioning in the release notes, add
`add-to-release-notes` label
- [ ] Changes that require user attention or interaction to upgrade are
documented in `docs/sources/setup/upgrade/_index.md`
- [ ] For Helm chart changes bump the Helm chart version in
`production/helm/loki/Chart.yaml` and update
`production/helm/loki/CHANGELOG.md` and
`production/helm/loki/README.md`. [Example
PR](d10549e)

Co-authored-by: Kaviraj Kanagaraj <[email protected]>

CHANGELOG.md

@@ -51,7 +51,6 @@
 * [9693](https://github.com/grafana/loki/pull/9693) **salvacorts** Add `keep` stage to LogQL.
 * [7447](https://github.com/grafana/loki/pull/7447) **ashwanthgoli** compactor: multi-store support.
 * [7754](https://github.com/grafana/loki/pull/7754) **ashwanthgoli** index-shipper: add support for multiple stores.
-* [8662](https://github.com/grafana/loki/pull/8662) **liguozhong**: LogQL: Introduce `distinct`
 * [9813](https://github.com/grafana/loki/pull/9813) **jeschkies**: Enable Protobuf encoding via content negotiation between querier and query frontend.
 * [10281](https://github.com/grafana/loki/pull/10281) **dannykopping**: Track effectiveness of hedged requests.
 

docs/sources/query/log_queries/_index.md

@@ -272,22 +272,6 @@ To evaluate the logical `and` first, use parenthesis, as in this example:
 
 Label filter expressions have support matching IP addresses. See [Matching IP addresses]({{< relref "../ip" >}}) for details.
 
-### Distinct filter expression
-
-Distinct filter expression allows filtering log lines using their original and extracted labels to filter out duplicate label values. The first line occurrence of a distinct value is returned, and the others are dropped.
-
-For example, for the following log lines:
-```log
-{"event": "access", "id": "1", "time": "2023-02-28 15:12:11"}
-{"event": "access", "id": "1", "time": "2023-02-28 15:13:11"}
-{"event": "access", "id": "2", "time": "2023-02-28 15:14:11"}
-{"event": "access", "id": "2", "time": "2023-02-28 15:15:11"}
-```
-The expression `{app="order"} | json | distinct id` will return the distinct occurrences of `id`:
-```log
-{"event": "access", "id": "1", "time": "2023-02-28 15:13:11"}
-{"event": "access", "id": "2", "time": "2023-02-28 15:15:11"}
-```
 ### Parser expression
 
 Parser expression can parse and extract labels from the log content. Those extracted labels can then be used for filtering using [label filter expressions](#label-filter-expression) or for [metric aggregations]({{< relref "../metric_queries" >}}).

pkg/logql/syntax/ast.go

@@ -275,7 +275,7 @@ func (e *PipelineExpr) Pipeline() (log.Pipeline, error) {
 func (e *PipelineExpr) HasFilter() bool {
 	for _, p := range e.MultiStages {
 		switch p.(type) {
-		case *LineFilterExpr, *LabelFilterExpr, *DistinctFilterExpr:
+		case *LineFilterExpr, *LabelFilterExpr:
 			return true
 		default:
 			continue
@@ -732,37 +732,6 @@ func (j *JSONExpressionParser) String() string {
 	return sb.String()
 }
 
-type DistinctFilterExpr struct {
-	labels []string
-	implicit
-}
-
-func newDistinctFilterExpr(labels []string) *DistinctFilterExpr {
-	return &DistinctFilterExpr{
-		labels: labels,
-	}
-}
-
-func (e *DistinctFilterExpr) Shardable() bool { return false }
-
-func (e *DistinctFilterExpr) Walk(f WalkFn) { f(e) }
-
-func (e *DistinctFilterExpr) Stage() (log.Stage, error) {
-	return log.NewDistinctFilter(e.labels)
-}
-
-func (e *DistinctFilterExpr) String() string {
-	var sb strings.Builder
-	sb.WriteString(fmt.Sprintf("%s %s ", OpPipe, OpFilterDistinct))
-	for i, label := range e.labels {
-		sb.WriteString(label)
-		if i+1 != len(e.labels) {
-			sb.WriteString(",")
-		}
-	}
-	return sb.String()
-}
-
 type internedStringSet map[string]struct {
 	s  string
 	ok bool
@@ -1036,8 +1005,6 @@ const (
 	// function filters
 	OpFilterIP = "ip"
 
-	OpFilterDistinct = "distinct"
-
 	// drop labels
 	OpDrop = "drop"
 

pkg/logql/syntax/ast_test.go

@@ -46,8 +46,6 @@ func Test_logSelectorExpr_String(t *testing.T) {
 		{`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b=ip("127.0.0.1") | level="error" | c=ip("::1")`, true}, // chain inside label filters.
 		{`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | regexp "(?P<foo>foo|bar)"`, true},
 		{`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | regexp "(?P<foo>foo|bar)" | ( ( foo<5.01 , bar>20ms ) or foo="bar" ) | line_format "blip{{.boop}}bap" | label_format foo=bar,bar="blip{{.blop}}"`, true},
-		{`{foo="bar"} | distinct id`, true},
-		{`{foo="bar"} | distinct id,time`, true},
 	}
 
 	for _, tt := range tests {
@@ -378,13 +376,6 @@ func Test_FilterMatcher(t *testing.T) {
 			},
 			[]linecheck{{"duration=5m total_bytes=5kB", true}, {"duration=1s total_bytes=256B", false}, {"duration=0s", false}},
 		},
-		{
-			`{app="foo"} | logfmt | distinct id`,
-			[]*labels.Matcher{
-				mustNewMatcher(labels.MatchEqual, "app", "foo"),
-			},
-			[]linecheck{{"id=foo", true}, {"id=foo", false}, {"id=bar", true}},
-		},
 	} {
 		tt := tt
 		t.Run(tt.q, func(t *testing.T) {

pkg/logql/syntax/expr.y

@@ -42,9 +42,7 @@ import (
   LogfmtParser            *LogfmtParserExpr
   LineFilters             *LineFilterExpr
   LineFilter              *LineFilterExpr
-  DistinctLabel           []string
   ParserFlags             []string
-  DistinctFilter          *DistinctFilterExpr
   PipelineExpr            MultiStageExpr
   PipelineStage           StageExpr
   BytesFilter             log.LabelFilterer
@@ -110,8 +108,6 @@ import (
 %type <LabelFilter>           labelFilter
 %type <LineFilters>           lineFilters
 %type <LineFilter>            lineFilter
-%type <DistinctFilter>        distinctFilter
-%type <DistinctLabel>         distinctLabel
 %type <ParserFlags>           parserFlags
 %type <LineFormatExpr>        lineFormatExpr
 %type <DecolorizeExpr>        decolorizeExpr
@@ -138,7 +134,7 @@ import (
 %token <duration> DURATION RANGE
 %token <val>      MATCHERS LABELS EQ RE NRE OPEN_BRACE CLOSE_BRACE OPEN_BRACKET CLOSE_BRACKET COMMA DOT PIPE_MATCH PIPE_EXACT
                   OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE RATE_COUNTER SUM SORT SORT_DESC AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK
-                  BYTES_OVER_TIME BYTES_RATE BOOL JSON DISTINCT REGEXP LOGFMT PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME
+                  BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME
                   MAX_OVER_TIME STDVAR_OVER_TIME STDDEV_OVER_TIME QUANTILE_OVER_TIME BYTES_CONV DURATION_CONV DURATION_SECONDS_CONV
                   FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME VECTOR LABEL_REPLACE UNPACK OFFSET PATTERN IP ON IGNORING GROUP_LEFT GROUP_RIGHT
                   DECOLORIZE DROP KEEP
@@ -282,8 +278,7 @@ pipelineStage:
   | PIPE labelFormatExpr         { $$ = $2 }
   | PIPE dropLabelsExpr          { $$ = $2 }
   | PIPE keepLabelsExpr          { $$ = $2 }
-  | PIPE distinctFilter          { $$ = $2 }
- ;
+  ;
 
 filterOp:
   IP { $$ = OpFilterIP }
@@ -342,14 +337,6 @@ labelsFormat:
 labelFormatExpr:
       LABEL_FMT labelsFormat { $$ = newLabelFmtExpr($2) };
 
-distinctLabel:
-    IDENTIFIER                     { $$ = []string{ $1 } }
-  | distinctLabel COMMA IDENTIFIER { $$ = append($1, $3) }
-  ;
-
-distinctFilter:
-      DISTINCT distinctLabel { $$ = newDistinctFilterExpr($2) };
-
 labelFilter:
       matcher                                        { $$ = log.NewStringLabelFilter($1) }
     | ipLabelFilter                                  { $$ = $1 }