simplify metrics test code

SeldonIO · Mar 27, 2020 · 6987fe2 · 6987fe2
1 parent be6969c
commit 6987fe2
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 180 deletions.
diff --git a/python/seldon_core/seldon_methods.py b/python/seldon_core/seldon_methods.py
@@ -30,7 +30,6 @@
 logger = logging.getLogger(__name__)
 
 
-
 def handle_raw_custom_metrics(response, seldon_metrics, is_proto):
     if is_proto:
         metrics = seldon_message_to_json(response.meta).get("metrics", [])

diff --git a/python/tests/test_metrics.py b/python/tests/test_metrics.py
@@ -299,6 +299,17 @@ def route_raw(self, msg):
         return request
 
 
+def verify_seldon_metrics(data, mycounter_value, histogram_entries):
+    assert data["GAUGE", "mygauge"]["value"] == 100
+    assert data["GAUGE", "customtag"]["value"] == 200
+    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
+    assert data["COUNTER", "mycounter"]["value"] == mycounter_value
+    assert np.allclose(
+        np.histogram(histogram_entries, BINS)[0], data["TIMER", "mytimer"]["value"][0]
+    )
+    assert np.allclose(data["TIMER", "mytimer"]["value"][1], np.sum(histogram_entries))
+
+
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevel])
 def test_seldon_metrics_predict(cls):
     user_object = cls()
@@ -311,28 +322,13 @@ def test_seldon_metrics_predict(cls):
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     rv = client.get('/predict?json={"data": {"names": ["input"], "ndarray": ["data"]}}')
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevel])
@@ -349,30 +345,15 @@ def test_seldon_metrics_aggregate(cls):
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     rv = client.get(
         '/aggregate?json={"seldonMessages": [{"data": {"names": ["input"], "ndarray": ["data"]}}]}'
     )
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevel])
@@ -389,30 +370,15 @@ def test_seldon_metrics_transform_input(cls):
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     rv = client.get(
         '/transform-input?json={"data": {"names": ["input"], "ndarray": ["data"]}}'
     )
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevel])
@@ -429,30 +395,15 @@ def test_seldon_metrics_transform_output(cls):
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     rv = client.get(
         '/transform-output?json={"data": {"names": ["input"], "ndarray": ["data"]}}'
     )
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevel])
@@ -467,28 +418,13 @@ def test_seldon_metrics_route(cls):
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     rv = client.get('/route?json={"data": {"names": ["input"], "ndarray": ["data"]}}')
     assert rv.status_code == 200
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevelGrpc])
@@ -502,30 +438,14 @@ def test_proto_seldon_metrics_predict(cls):
     )
 
     request = prediction_pb2.SeldonMessage(data=datadef)
-    app.Predict(request, None)
 
+    app.Predict(request, None)
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     app.Predict(request, None)
-
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevelGrpc])
@@ -547,30 +467,14 @@ def test_proto_seldon_metrics_aggregate(cls):
     msg2 = prediction_pb2.SeldonMessage(data=datadef2)
 
     request = prediction_pb2.SeldonMessageList(seldonMessages=[msg1, msg2])
-    app.Aggregate(request, None)
 
+    app.Aggregate(request, None)
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     app.Aggregate(request, None)
-
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevelGrpc])
@@ -584,30 +488,14 @@ def test_proto_seldon_metrics_transform_input(cls):
     )
 
     request = prediction_pb2.SeldonMessage(data=datadef)
-    app.TransformInput(request, None)
 
+    app.TransformInput(request, None)
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     app.TransformInput(request, None)
-
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevelGrpc])
@@ -621,30 +509,14 @@ def test_proto_seldon_metrics_transform_output(cls):
     )
 
     request = prediction_pb2.SeldonMessage(data=datadef)
-    app.TransformOutput(request, None)
 
+    app.TransformOutput(request, None)
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
+    verify_seldon_metrics(data, 1, [0.0202])
 
     app.TransformOutput(request, None)
-
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevelGrpc])
@@ -661,27 +533,11 @@ def test_proto_seldon_metrics_route(cls):
     app.Route(request, None)
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 1
-    assert np.allclose(
-        np.histogram([20.2 / 1000], BINS)[0], data["TIMER", "mytimer"]["value"][0]
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0202)
-
+    verify_seldon_metrics(data, 1, [0.0202])
     app.Route(request, None)
 
     data = seldon_metrics.data[os.getpid()]
-    assert data["GAUGE", "mygauge"]["value"] == 100
-    assert data["GAUGE", "customtag"]["value"] == 200
-    assert data["GAUGE", "customtag"]["tags"] == {"mytag": "mytagvalue"}
-    assert data["COUNTER", "mycounter"]["value"] == 2
-    assert np.allclose(
-        np.histogram([20.2 / 1000, 20.2 / 1000], BINS)[0],
-        data["TIMER", "mytimer"]["value"][0],
-    )
-    assert np.allclose(data["TIMER", "mytimer"]["value"][1], 0.0404)
+    verify_seldon_metrics(data, 2, [0.0202, 0.0202])
 
 
 @pytest.mark.parametrize("cls", [UserObject, UserObjectLowLevel])