Allow numbers and boolean values for tags. (#1712)

* Allow numbers and boolean values for tags. Store numbers as scaled_floats, scaling_factor 1000000 * Update tags in metricset to behave the same way as transaction.tags fixes #828.
elastic · Jan 9, 2019 · a2e6d88 · a2e6d88
1 parent 09926a4
commit a2e6d88
Show file tree

Hide file tree

Showing 29 changed files with 93 additions and 66 deletions.
diff --git a/_meta/fields.common.yml b/_meta/fields.common.yml
@@ -33,7 +33,11 @@
 
         - name: tags
           type: object
-          object_type: keyword
+          object_type_params:
+            - object_type: keyword
+            - object_type: boolean
+            - object_type: scaled_float
+              scaling_factor: 1000000
           dynamic: true
           description: >
             A flat mapping of user-defined tags with string values.

diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc
@@ -6,6 +6,11 @@
 
 - Update Go version to 1.11.4 {pull}1700[1700].
 - Update Elastic APM Go agent to v1.1.2 {pull}1711[1711], {pull}1728[1728].
+- Allow numbers and boolean values for `transaction.tags`, `span.tags`, `metricset.tags` {pull}1712[1712].
+
+[float]
+==== Removed
+- Remove support for using dots in tags for experimental metricset endpoint {pull}1712[1712].
 
 [float]
 ==== Removed

diff --git a/docs/data/elasticsearch/generated/metricsets.json b/docs/data/elasticsearch/generated/metricsets.json
@@ -18,8 +18,9 @@
                     "name": "1234_service-12a3"
                 },
                 "tags": {
-                    "code": "200",
-                    "some.other.code": "abc"
+                    "code": 200,
+                    "some": "abc",
+                    "success": true
                 }
             },
             "dotted": {

diff --git a/docs/data/elasticsearch/generated/spans.json b/docs/data/elasticsearch/generated/spans.json
@@ -86,7 +86,10 @@
                     "name": "backendspans"
                 },
                 "tags": {
-                    "tag1": "value1"
+                    "tag1": "value1",
+                    "tag2": 123,
+                    "tag3": 12.34,
+                    "tag4": true
                 }
             },
             "parent": {

diff --git a/docs/data/elasticsearch/generated/transactions.json b/docs/data/elasticsearch/generated/transactions.json
@@ -168,7 +168,10 @@
                     "platform": "darwin"
                 },
                 "tags": {
-                    "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8"
+                    "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8",
+                    "tag2": 12,
+                    "tag3": 12.45,
+                    "tag4": false
                 },
                 "user": {
                     "email": "[email protected]",

diff --git a/docs/spec/metricsets/metricset.json b/docs/spec/metricsets/metricset.json
@@ -19,21 +19,7 @@
                     "additionalProperties": false
                 },
                 "tags": {
-                    "type": [
-                        "object",
-                        "null"
-                    ],
-                    "description": "A flat mapping of user-defined tags with string values.",
-                    "patternProperties": {
-                        "^[^*\"]*$": {
-                            "type": [
-                                "string",
-                                "null"
-                            ],
-                            "maxLength": 1024
-                        }
-                    },
-                    "additionalProperties": false
+                    "$ref": "../tags.json"
                 }
             },
             "required": ["samples"]

diff --git a/docs/spec/tags.json b/docs/spec/tags.json
@@ -2,10 +2,10 @@
     "$id": "doc/spec/tags.json",
     "title": "Tags",
     "type": ["object", "null"],
-    "description": "A flat mapping of user-defined tags with string values.",
+    "description": "A flat mapping of user-defined tags with string, boolean or number values.",
     "patternProperties": {
         "^[^.*\"]*$": {
-            "type": ["string", "null"],
+            "type": ["string", "boolean", "number", "null"],
             "maxLength": 1024
         }
     },

diff --git a/include/fields.go b/include/fields.go
diff --git a/model/error/generated/schema/error.go b/model/error/generated/schema/error.go
@@ -213,10 +213,10 @@ const ModelSchema = `{
                 "$id": "doc/spec/tags.json",
     "title": "Tags",
     "type": ["object", "null"],
-    "description": "A flat mapping of user-defined tags with string values.",
+    "description": "A flat mapping of user-defined tags with string, boolean or number values.",
     "patternProperties": {
         "^[^.*\"]*$": {
-            "type": ["string", "null"],
+            "type": ["string", "boolean", "number", "null"],
             "maxLength": 1024
         }
     },

diff --git a/model/metricset/generated/schema/metricset.go b/model/metricset/generated/schema/metricset.go
@@ -54,21 +54,17 @@ const ModelSchema = `{
                     "additionalProperties": false
                 },
                 "tags": {
-                    "type": [
-                        "object",
-                        "null"
-                    ],
-                    "description": "A flat mapping of user-defined tags with string values.",
-                    "patternProperties": {
-                        "^[^*\"]*$": {
-                            "type": [
-                                "string",
-                                "null"
-                            ],
-                            "maxLength": 1024
-                        }
-                    },
-                    "additionalProperties": false
+                        "$id": "doc/spec/tags.json",
+    "title": "Tags",
+    "type": ["object", "null"],
+    "description": "A flat mapping of user-defined tags with string, boolean or number values.",
+    "patternProperties": {
+        "^[^.*\"]*$": {
+            "type": ["string", "boolean", "number", "null"],
+            "maxLength": 1024
+        }
+    },
+    "additionalProperties": false
                 }
             },
             "required": ["samples"]

diff --git a/model/span/generated/schema/span.go b/model/span/generated/schema/span.go
@@ -117,10 +117,10 @@ const ModelSchema = `{
                                 "$id": "doc/spec/tags.json",
     "title": "Tags",
     "type": ["object", "null"],
-    "description": "A flat mapping of user-defined tags with string values.",
+    "description": "A flat mapping of user-defined tags with string, boolean or number values.",
     "patternProperties": {
         "^[^.*\"]*$": {
-            "type": ["string", "null"],
+            "type": ["string", "boolean", "number", "null"],
             "maxLength": 1024
         }
     },

diff --git a/model/transaction/generated/schema/transaction.go b/model/transaction/generated/schema/transaction.go
@@ -214,10 +214,10 @@ const ModelSchema = `{
                 "$id": "doc/spec/tags.json",
     "title": "Tags",
     "type": ["object", "null"],
-    "description": "A flat mapping of user-defined tags with string values.",
+    "description": "A flat mapping of user-defined tags with string, boolean or number values.",
     "patternProperties": {
         "^[^.*\"]*$": {
-            "type": ["string", "null"],
+            "type": ["string", "boolean", "number", "null"],
             "maxLength": 1024
         }
     },

diff --git a/processor/stream/package_tests/error_attrs_test.go b/processor/stream/package_tests/error_attrs_test.go
@@ -198,10 +198,10 @@ func TestPayloadDataForError(t *testing.T) {
 			{Key: "error.context.request.cookies", Valid: val{obj{}},
 				Invalid: []tests.Invalid{{Msg: `/context/properties/request/properties/cookies/type`, Values: val{102, "a"}}}},
 			{Key: "error.context.tags",
-				Valid: val{obj{tests.Str1024Special: tests.Str1024Special}},
+				Valid: val{obj{tests.Str1024Special: tests.Str1024Special}, obj{tests.Str1024: 123.45}, obj{tests.Str1024: true}},
 				Invalid: []tests.Invalid{
 					{Msg: `context/properties/tags/type`, Values: val{"tags"}},
-					{Msg: `context/properties/tags/patternproperties`, Values: val{obj{"invalid": tests.Str1025}, obj{tests.Str1024: 123}, obj{tests.Str1024: obj{}}}},
+					{Msg: `context/properties/tags/patternproperties`, Values: val{obj{"invalid": tests.Str1025}, obj{tests.Str1024: obj{}}}},
 					{Msg: `context/properties/tags/additionalproperties`, Values: val{obj{"invali*d": "hello"}, obj{"invali\"d": "hello"}, obj{"invali.d": "hello"}}}}},
 			{Key: "error.context.user.id", Valid: val{123, tests.Str1024Special},
 				Invalid: []tests.Invalid{

diff --git a/processor/stream/package_tests/metricset_attrs_test.go b/processor/stream/package_tests/metricset_attrs_test.go
@@ -63,10 +63,10 @@ func TestInvalidPayloads(t *testing.T) {
 			Invalid: []tests.Invalid{
 				{Msg: `timestamp/type`, Values: val{"1496170422281000"}}}},
 		{Key: "metricset.tags",
-			Valid: val{obj{tests.Str1024Special: tests.Str1024Special}},
+			Valid: val{obj{tests.Str1024Special: tests.Str1024Special}, obj{tests.Str1024: 123.45}, obj{tests.Str1024: true}},
 			Invalid: []tests.Invalid{
 				{Msg: `tags/type`, Values: val{"tags"}},
-				{Msg: `tags/patternproperties`, Values: val{obj{"invalid": tests.Str1025}, obj{tests.Str1024: 123}, obj{tests.Str1024: obj{}}}},
+				{Msg: `tags/patternproperties`, Values: val{obj{"invalid": tests.Str1025}, obj{tests.Str1024: obj{}}}},
 				{Msg: `tags/additionalproperties`, Values: val{obj{"invali*d": "hello"}, obj{"invali\"d": "hello"}}}},
 		},
 		{

diff --git a/processor/stream/package_tests/span_attrs_test.go b/processor/stream/package_tests/span_attrs_test.go
@@ -46,7 +46,6 @@ func spanPayloadAttrsNotInFields() *tests.Set {
 		"context.http",
 		"context.http.url",
 		"context.http.method",
-		"context.tags.tag1",
 	)
 }
 
@@ -67,7 +66,7 @@ func spanPayloadAttrsNotInJsonSchema() *tests.Set {
 	return tests.NewSet(
 		"span",
 		"span.stacktrace.vars.key",
-		"span.context.tags.tag1",
+		tests.Group("span.context.tags.tag"),
 	)
 }
 
@@ -165,10 +164,10 @@ func TestPayloadDataForSpans(t *testing.T) {
 	spanProcSetup().DataValidation(t,
 		[]tests.SchemaTestData{
 			{Key: "span.context.tags",
-				Valid: val{obj{tests.Str1024Special: tests.Str1024Special}},
+				Valid: val{obj{tests.Str1024Special: tests.Str1024Special}, obj{tests.Str1024: 123.45}, obj{tests.Str1024: true}},
 				Invalid: []tests.Invalid{
 					{Msg: `tags/type`, Values: val{"tags"}},
-					{Msg: `tags/patternproperties`, Values: val{obj{"invalid": tests.Str1025}, obj{tests.Str1024: 123}, obj{tests.Str1024: obj{}}}},
+					{Msg: `tags/patternproperties`, Values: val{obj{"invalid": tests.Str1025}, obj{tests.Str1024: obj{}}}},
 					{Msg: `tags/additionalproperties`, Values: val{obj{"invali*d": "hello"}, obj{"invali\"d": "hello"}, obj{"invali.d": "hello"}}}},
 			},
 			{Key: "span.timestamp",

diff --git a/processor/stream/package_tests/transaction_attrs_test.go b/processor/stream/package_tests/transaction_attrs_test.go
@@ -173,10 +173,10 @@ func TestPayloadDataForTransaction(t *testing.T) {
 				Valid:   []interface{}{obj{}},
 				Invalid: []tests.Invalid{{Msg: `context/properties/request/properties/cookies/type`, Values: val{123, ""}}}},
 			{Key: "transaction.context.tags",
-				Valid: val{obj{tests.Str1024Special: tests.Str1024Special}},
+				Valid: val{obj{tests.Str1024Special: tests.Str1024Special}, obj{tests.Str1024: 123.45}, obj{tests.Str1024: true}},
 				Invalid: []tests.Invalid{
 					{Msg: `tags/type`, Values: val{"tags"}},
-					{Msg: `tags/patternproperties`, Values: val{obj{"invalid": tests.Str1025}, obj{tests.Str1024: 123}, obj{tests.Str1024: obj{}}}},
+					{Msg: `tags/patternproperties`, Values: val{obj{"invalid": tests.Str1025}, obj{tests.Str1024: obj{}}}},
 					{Msg: `tags/additionalproperties`, Values: val{obj{"invali*d": "hello"}, obj{"invali\"d": "hello"}, obj{"invali.d": "hello"}}}}},
 			{Key: "transaction.context.user.id",
 				Valid: val{123, tests.Str1024Special},

diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationMetricsets.approved.json
@@ -18,8 +18,9 @@
                     "name": "1234_service-12a3"
                 },
                 "tags": {
-                    "code": "200",
-                    "some.other.code": "abc"
+                    "code": 200,
+                    "some": "abc",
+                    "success": true
                 }
             },
             "dotted": {

diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json
@@ -86,7 +86,10 @@
                     "name": "backendspans"
                 },
                 "tags": {
-                    "tag1": "value1"
+                    "tag1": "value1",
+                    "tag2": 123,
+                    "tag3": 12.34,
+                    "tag4": true
                 }
             },
             "parent": {

diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json
@@ -168,7 +168,10 @@
                     "platform": "darwin"
                 },
                 "tags": {
-                    "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8"
+                    "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8",
+                    "tag2": 12,
+                    "tag3": 12.45,
+                    "tag4": false
                 },
                 "user": {
                     "email": "[email protected]",

diff --git a/testdata/intake-v2/invalid-json-metadata.ndjson b/testdata/intake-v2/invalid-json-metadata.ndjson
@@ -1,2 +1,2 @@
 {"metadata": {"invalid-json"}}
-{"metricset": { "samples": { "byte_counter": { "value": 1 }, "short_counter": { "value": 227 }, "integer_gauge": { "value": 42767 }, "long_gauge": { "value": 3147483648 }, "float_gauge": { "value": 9.16 }, "double_gauge": { "value": 3.141592653589793 }, "dotted.float.gauge": { "value": 6.12 }, "negative.d.o.t.t.e.d": { "value": -1022 } }, "tags": { "code": "200", "some.other.code": "abc" }, "timestamp": 1496170422281000 }}
+{"metricset": { "samples": { "byte_counter": { "value": 1 }, "short_counter": { "value": 227 }, "integer_gauge": { "value": 42767 }, "long_gauge": { "value": 3147483648 }, "float_gauge": { "value": 9.16 }, "double_gauge": { "value": 3.141592653589793 }, "dotted.float.gauge": { "value": 6.12 }, "negative.d.o.t.t.e.d": { "value": -1022 } }, "tags": { "code": "200", "code": "abc" }, "timestamp": 1496170422281000 }}
diff --git a/testdata/intake-v2/invalid-metadata-2.ndjson b/testdata/intake-v2/invalid-metadata-2.ndjson
@@ -1,3 +1,3 @@
 {"not": "metadata"}
-{"metricset": { "samples": { "byte_counter": { "value": 1 }, "short_counter": { "value": 227 }, "integer_gauge": { "value": 42767 }, "long_gauge": { "value": 3147483648 }, "float_gauge": { "value": 9.16 }, "double_gauge": { "value": 3.141592653589793 }, "dotted.float.gauge": { "value": 6.12 }, "negative.d.o.t.t.e.d": { "value": -1022 } }, "tags": { "code": "200", "some.other.code": "abc" }, "timestamp": 1496170422281000 }}
+{"metricset": { "samples": { "byte_counter": { "value": 1 }, "short_counter": { "value": 227 }, "integer_gauge": { "value": 42767 }, "long_gauge": { "value": 3147483648 }, "float_gauge": { "value": 9.16 }, "double_gauge": { "value": 3.141592653589793 }, "dotted.float.gauge": { "value": 6.12 }, "negative.d.o.t.t.e.d": { "value": -1022 } }, "tags": { "code": "200", "key": "abc" }, "timestamp": 1496170422281000 }}
 
diff --git a/testdata/intake-v2/invalid-metadata.ndjson b/testdata/intake-v2/invalid-metadata.ndjson
@@ -1,3 +1,3 @@
 {"metadata": {"user": null}}
-{"metricset": { "samples": { "byte_counter": { "value": 1 }, "short_counter": { "value": 227 }, "integer_gauge": { "value": 42767 }, "long_gauge": { "value": 3147483648 }, "float_gauge": { "value": 9.16 }, "double_gauge": { "value": 3.141592653589793 }, "dotted.float.gauge": { "value": 6.12 }, "negative.d.o.t.t.e.d": { "value": -1022 } }, "tags": { "code": "200", "some.other.code": "abc" }, "timestamp": 1496170422281000 }}
+{"metricset": { "samples": { "byte_counter": { "value": 1 }, "short_counter": { "value": 227 }, "integer_gauge": { "value": 42767 }, "long_gauge": { "value": 3147483648 }, "float_gauge": { "value": 9.16 }, "double_gauge": { "value": 3.141592653589793 }, "dotted.float.gauge": { "value": 6.12 }, "negative.d.o.t.t.e.d": { "value": -1022 } }, "tags": { "code": "200", "some": "abc" }, "timestamp": 1496170422281000 }}
 
diff --git a/testdata/intake-v2/metricsets.ndjson b/testdata/intake-v2/metricsets.ndjson
@@ -1,3 +1,3 @@
 {"metadata": {"user": null, "process": {"ppid": null, "pid": 1234, "argv": null, "title": null}, "system": null, "service": {"name": "1234_service-12a3", "language": {"version": null, "name":"ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node"}, "environment": null, "framework": null,"version": null, "runtime": null}}}
-{"metricset": { "samples": { "byte_counter": { "value": 1 }, "short_counter": { "value": 227 }, "integer_gauge": { "value": 42767 }, "long_gauge": { "value": 3147483648 }, "float_gauge": { "value": 9.16 }, "double_gauge": { "value": 3.141592653589793 }, "dotted.float.gauge": { "value": 6.12 }, "negative.d.o.t.t.e.d": { "value": -1022 } }, "tags": { "code": "200", "some.other.code": "abc" }, "timestamp": 1496170422281000 }}
+{"metricset": { "samples": { "byte_counter": { "value": 1 }, "short_counter": { "value": 227 }, "integer_gauge": { "value": 42767 }, "long_gauge": { "value": 3147483648 }, "float_gauge": { "value": 9.16 }, "double_gauge": { "value": 3.141592653589793 }, "dotted.float.gauge": { "value": 6.12 }, "negative.d.o.t.t.e.d": { "value": -1022 } }, "tags": { "some": "abc", "code": 200, "success": true }, "timestamp": 1496170422281000 }}
 { "metricset": { "samples": { "go.memstats.heap.sys.bytes": { "value": 6.520832e+06 } }, "timestamp": 1496170422281000 }}
diff --git a/testdata/intake-v2/spans.ndjson b/testdata/intake-v2/spans.ndjson
@@ -1,6 +1,6 @@
 {"metadata": {"user": {"id": "123", "email": "[email protected]", "username": "john"}, "process": {"ppid": 6789, "pid": 1234,"argv": ["node", "server.js"], "title": "node"}, "system": {"platform": "darwin", "hostname": "prod1.example.com", "architecture": "x64"}, "service": {"name": "backendspans", "language": {"version": "8", "name": "ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node"}, "environment": "staging", "framework": {"version": "1.2.3", "name": "Express"}, "version": "5.1.3", "runtime": {"version": "8.0.0", "name": "node"}}}}
 {"span": {"trace_id": "fdedef0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "abcdef01234567", "transaction_id": "01af25874dec69dd", "name": "GET /api/types", "type": "request","start": null, "duration": 141.581, "timestamp": 1532976822281000}}
 { "span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "0000000011111111", "id": "1234abcdef567895", "transaction_id": "ab45781d265894fe", "name": "GET /api/types", "type": "request", "start": 22, "duration": 32.592981, "timestamp": 1532976822281000} }
-{ "span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdefabcdef7890", "id": "0123456a89012345", "transaction_id": "ab23456a89012345", "name": "GET /api/types", "type": "request", "start": 1.845, "duration": 3.5642981, "stacktrace": [], "context": {"tags": {"tag1": "value1"}}  }}
+{ "span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdefabcdef7890", "id": "0123456a89012345", "transaction_id": "ab23456a89012345", "name": "GET /api/types", "type": "request", "start": 1.845, "duration": 3.5642981, "stacktrace": [], "context": {"tags": {"tag1": "value1", "tag2": 123, "tag3": 12.34, "tag4": true, "tag5": null}}  }}
 { "span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "ababcdcdefefabde", "id": "abcde56a89012345", "transaction_id": "bed3456a89012345", "name": "get /api/types",  "sync": false, "type": "request", "subtype": "http", "action": "call", "start": 0, "duration": 13.9802981, "stacktrace": null, "context": null }}
 { "span": {"trace_id": "abcdef0123456789abcdef9876543210", "parent_id": "abcdef0123456789", "id": "1234567890aaaade","transaction_id": "aff4567890aaaade", "sync": true, "name": "SELECT FROM product_types", "type": "db", "subtype": "postgresql", "action": "query", "start": 2.83092, "duration": 3.781912, "stacktrace": [{ "filename": "net.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ "    ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ "  var trans = this.currentTransaction", "" ], "post_context": [ "    ins.currentTransaction = prev", "    return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user" }, "http": { "url": "http://localhost:8000", "status_code": 200, "method": "GET" } } }}