Skip to content

Commit

Permalink
[query] add specialized matchers for empty EQ/NEQ matchers (#1986)
Browse files Browse the repository at this point in the history
  • Loading branch information
arnikola authored Oct 23, 2019
1 parent 71f7318 commit e3d0f4f
Show file tree
Hide file tree
Showing 4 changed files with 173 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,27 @@

set -ex
source $GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/common.sh
t=$(date +%s)

function write_metrics {
NUM=$1
EXTRA=${2:-default}
echo "Writing $NUM metrics to [0.0.0.0:9003]"
# set +x
set +x
for (( i=0; i<$NUM; i++ ))
do
curl -X POST 0.0.0.0:9003/writetagged -d '{
"namespace": "unagg",
"id": "{__name__=\"'$METRIC_NAME'\",val=\"'$i'\"}",
"id": "{__name__=\"'$METRIC_NAME'\",'$EXTRA'=\"extra\",val=\"'$i'\"}",
"tags": [
{
"name": "__name__",
"value": "'$METRIC_NAME'"
},
{
"name": "'$EXTRA'",
"value": "extra"
},
{
"name": "val",
"value": "'$i'"
Expand All @@ -28,23 +34,21 @@ function write_metrics {
}
}'
done
# set -x
set -x
}

function test_instantaneous {
QUERY=$1
EXPECTED_COUNT=$2
EXPECTED=$3
RESPONSE=$(curl -sSL "http://localhost:7201/api/v1/query?query=$QUERY")
echo $REPONSE | jq .data.result
ACTUAL_COUNT=$(echo $RESPONSE | jq '.data.result | length')
ACTUAL=$(echo $RESPONSE | jq .data.result[].metric.foo | tr -d "\n")
CONCAT=$(echo $EXPECTED | tr -d " ")
test $ACTUAL_COUNT = $EXPECTED_COUNT && test $ACTUAL = $CONCAT
}

function test_replace {
export t=$(date +%s)
function test_replace {
METRIC_NAME="quail_$t"
write_metrics 5
sleep 1
Expand All @@ -54,7 +58,31 @@ function test_replace {
test_instantaneous $query 5 "\"bar_0\" \"bar_1\" \"bar_2\" \"bar_3\" \"bar_4\""
}

function test_parse_query {
function test_exists {
QUERY=$1
EXPECTED_EXISTS=$2
EXPECTED_NOT_EXISTS=$3
EXPECTED_COUNT=$4
echo $QUERY "IS METRIC NAME"
RESPONSE=$(curl -sSL "http://localhost:7201/api/v1/query?query=$METRIC_NAME\{$QUERY\}")
ACTUAL_COUNT_EXISTS=$(echo $RESPONSE | jq .data.result[].metric.$EXPECTED_EXISTS | grep extra | wc -l)
ACTUAL_COUNT_NOT_EXISTS=$(echo $RESPONSE | jq .data.result[].metric.$EXPECTED_NOT_EXISTS | grep extra | wc -l)
test $ACTUAL_COUNT_EXISTS = $EXPECTED_COUNT && test $ACTUAL_COUNT_NOT_EXISTS = 0
}

function test_empty_matcher {
export METRIC_NAME="foo_$t"
write_metrics 5 exists
write_metrics 5 not_exists

retry_with_backoff ATTEMPTS=3 TIMEOUT=1 test_exists 'not_exists=\"\"' exists not_exists 5
retry_with_backoff ATTEMPTS=3 TIMEOUT=1 test_exists 'not_exists!=\"\"' not_exists exists 5

retry_with_backoff ATTEMPTS=3 TIMEOUT=1 test_exists 'exists=\"\"' not_exists exists 5
retry_with_backoff ATTEMPTS=3 TIMEOUT=1 test_exists 'exists!=\"\"' exists not_exists 5
}

function test_parse_threshold {
test $(curl 'http://localhost:7201/api/v1/parse?query=up' | jq .name) = '"fetch"'
THRESHOLD=$(curl 'http://localhost:7201/api/v1/threshold?query=up>1')
test $(echo $THRESHOLD | jq .threshold.comparator) = '">"'
Expand All @@ -64,10 +92,11 @@ function test_parse_query {
THRESHOLD=$(curl 'http://localhost:7201/api/v1/threshold?query=1>up')
test $(echo $THRESHOLD | jq .threshold.comparator) = '"<"'
test $(echo $THRESHOLD | jq .threshold.value) = 1
test $(echo $THRESHOLD | jq .query.name) = '"fetch"'
test $(echo $THRESHOLD | jq .query.name) = '"fetch"'
}

function test_correctness {
test_parse_query
test_parse_threshold
test_replace
test_empty_matcher
}
6 changes: 3 additions & 3 deletions scripts/docker-integration-tests/prometheus/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
set -xe

source $GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/common.sh
source $GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/prometheus/test_correctness.sh
source $GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/prometheus/test-correctness.sh
REVISION=$(git rev-parse HEAD)
COMPOSE_FILE=$GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/prometheus/docker-compose.yml
# quay.io/m3db/prometheus_remote_client_golang @ v0.4.3
Expand Down Expand Up @@ -115,13 +115,13 @@ function test_query_limits_applied {
# coordinator (limit set to 100 in m3coordinator.yml)
echo "Test query limit with coordinator defaults"
ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff \
'[[ $(curl -s 0.0.0.0:7201/api/v1/query?query=\\{name!=\"\"\\} | jq -r ".data.result | length") -eq 100 ]]'
'[[ $(curl -s 0.0.0.0:7201/api/v1/query?query=\\{__name__!=\"\"\\} | jq -r ".data.result | length") -eq 100 ]]'
# Test the default series limit applied when directly querying
# coordinator (limit set by header)
echo "Test query limit with coordinator limit header"
ATTEMPTS=50 TIMEOUT=2 MAX_TIMEOUT=4 retry_with_backoff \
'[[ $(curl -s -H "M3-Limit-Max-Series: 10" 0.0.0.0:7201/api/v1/query?query=\\{name!=\"\"\\} | jq -r ".data.result | length") -eq 10 ]]'
'[[ $(curl -s -H "M3-Limit-Max-Series: 10" 0.0.0.0:7201/api/v1/query?query=\\{__name__!=\"\"\\} | jq -r ".data.result | length") -eq 10 ]]'
}
function prometheus_query_native {
Expand Down
114 changes: 114 additions & 0 deletions src/query/parser/promql/matcher_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package promql

import (
"bytes"
"testing"

"github.com/m3db/m3/src/query/models"

"github.com/prometheus/prometheus/pkg/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestLabelMatchesToModelMatcher(t *testing.T) {
opts := models.NewTagOptions()

labels := []*labels.Matcher{
&labels.Matcher{
Type: labels.MatchEqual,
Name: "foo",
},
&labels.Matcher{
Type: labels.MatchEqual,
Name: "foo",
Value: "bar",
},
&labels.Matcher{
Type: labels.MatchNotEqual,
Name: "foo",
},
&labels.Matcher{
Type: labels.MatchNotEqual,
Name: "foo",
Value: "bar",
},
&labels.Matcher{
Type: labels.MatchRegexp,
Name: "foo",
Value: ".*",
},
&labels.Matcher{
Type: labels.MatchNotRegexp,
Name: "foo",
Value: ".*",
},
}

matchers, err := LabelMatchersToModelMatcher(labels, opts)
assert.NoError(t, err)

expected := models.Matchers{
models.Matcher{
Type: models.MatchNotField,
Name: []byte("foo"),
Value: []byte{},
},
models.Matcher{
Type: models.MatchEqual,
Name: []byte("foo"),
Value: []byte("bar"),
},
models.Matcher{
Type: models.MatchField,
Name: []byte("foo"),
Value: []byte{},
},
models.Matcher{
Type: models.MatchNotEqual,
Name: []byte("foo"),
Value: []byte("bar"),
},
models.Matcher{
Type: models.MatchRegexp,
Name: []byte("foo"),
Value: []byte(".*"),
},
models.Matcher{
Type: models.MatchNotRegexp,
Name: []byte("foo"),
Value: []byte(".*"),
},
}

require.Equal(t, len(expected), len(matchers))
equalish := func(a, b models.Matcher) bool {
return bytes.Equal(a.Name, b.Name) &&
bytes.Equal(a.Value, b.Value) &&
a.Type == b.Type
}

for i, ex := range expected {
assert.True(t, equalish(ex, matchers[i]))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -332,9 +332,9 @@ func LabelMatchersToModelMatcher(
lMatchers []*labels.Matcher,
tagOpts models.TagOptions,
) (models.Matchers, error) {
matchers := make(models.Matchers, len(lMatchers))
for i, m := range lMatchers {
modelType, err := promTypeToM3(m.Type)
matchers := make(models.Matchers, 0, len(lMatchers))
for _, m := range lMatchers {
matchType, err := promTypeToM3(m.Type)
if err != nil {
return nil, err
}
Expand All @@ -346,12 +346,25 @@ func LabelMatchersToModelMatcher(
name = []byte(m.Name)
}

match, err := models.NewMatcher(modelType, name, []byte(m.Value))
value := []byte(m.Value)
// NB: special case here since by Prometheus convention, a NEQ tag with no
// provided value is interpreted as verifying that the tag exists.
// Similarily, EQ tag with no provided value is interpreted as ensuring that
// the tag does not exist.
if len(value) == 0 {
if matchType == models.MatchNotEqual {
matchType = models.MatchField
} else if matchType == models.MatchEqual {
matchType = models.MatchNotField
}
}

match, err := models.NewMatcher(matchType, name, value)
if err != nil {
return nil, err
}

matchers[i] = match
matchers = append(matchers, match)
}

return matchers, nil
Expand Down

0 comments on commit e3d0f4f

Please sign in to comment.