Added end to end test for mem ballast

This commit adds an end to end test to verify performance gain acheived with memory ballast It mainly checks the CPU usage and does not test against memory usage. I'll be submitting another PR that will specifically test memory behaviour and ensure the ballast does not actually consume memory. `TestNoBackend10kSPS` test was very reliably consuming >20 CPU on my machine so I increased the accepted values to 30 for this test.
open-telemetry · Jul 3, 2019 · 541c6f3 · 541c6f3
1 parent c50224d
commit 541c6f3
Show file tree

Hide file tree

Showing 7 changed files with 136 additions and 61 deletions.
diff --git a/go.mod b/go.mod
@@ -6,24 +6,20 @@ require (
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	contrib.go.opencensus.io/exporter/zipkin v0.1.1
 	contrib.go.opencensus.io/resource v0.1.1
-	github.com/VividCortex/gohistogram v1.0.0 // indirect
 	github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7
-	github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b // indirect
 	github.com/census-instrumentation/opencensus-proto v0.2.0
 	github.com/go-kit/kit v0.8.0
-	github.com/gogo/googleapis v1.2.0 // indirect
 	github.com/golang/protobuf v1.3.1
 	github.com/google/go-cmp v0.3.0
 	github.com/gorilla/mux v1.6.2
 	github.com/grpc-ecosystem/grpc-gateway v1.9.0
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jaegertracing/jaeger v1.9.0
 	github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024
 	github.com/omnition/scribe-go v0.0.0-20190131012523-9e3c68f31124
+	github.com/open-telemetry/opentelemetry-service/testbed v0.0.0-20190702215910-6b130d62dbaa // indirect
 	github.com/openzipkin/zipkin-go v0.1.6
 	github.com/orijtech/prometheus-go-metrics-exporter v0.0.3-0.20190313163149-b321c5297f60
 	github.com/pkg/errors v0.8.0
-	github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a // indirect
 	github.com/prometheus/client_golang v0.9.3
 	github.com/prometheus/common v0.4.0
 	github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084
@@ -33,7 +29,6 @@ require (
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.4.0
-	github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect
 	github.com/stretchr/testify v1.3.0
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-lib v2.0.0+incompatible

diff --git a/go.sum b/go.sum
diff --git a/testbed/go.mod b/testbed/go.mod
@@ -4,9 +4,9 @@ go 1.12
 
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.1.1-0.20190430175949-e8b55949d948
+	github.com/gogo/googleapis v1.2.0 // indirect
 	github.com/open-telemetry/opentelemetry-service v0.0.0-20190625135304-4bd705a25a35
 	github.com/shirou/gopsutil v2.18.12+incompatible
-	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.3.0
 	go.opencensus.io v0.22.0

diff --git a/testbed/go.sum b/testbed/go.sum
@@ -185,6 +185,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/open-telemetry/opentelemetry-service/testbed v0.0.0-20190702215910-6b130d62dbaa/go.mod h1:55KQ0SPaonTFJxENtpPuZkIV853aovZ6bUtZYB9ROj0=
 github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.1 h1:IYN/cK5AaULfeMAlgFZSIBLSpsZ5MRHDy1fKBEqqJfQ=

diff --git a/testbed/testbed/test_case.go b/testbed/testbed/test_case.go
@@ -139,7 +139,7 @@ func (tc *TestCase) StartAgent(args ...string) {
 		name:         "Agent",
 		logFilePath:  logFileName,
 		cmd:          testBedConfig.Agent,
-		cmdArgs:      args, 
+		cmdArgs:      args,
 		resourceSpec: &tc.resourceSpec,
 	})
 
@@ -189,7 +189,7 @@ func (tc *TestCase) StopBackend() {
 	tc.MockBackend.Stop()
 }
 
-// AgentMemoryInfo returns raw memory info struct about the agent 
+// AgentMemoryInfo returns raw memory info struct about the agent
 // as returned by github.com/shirou/gopsutil/process
 func (tc *TestCase) AgentMemoryInfo() (uint32, uint32, error) {
 	stat, err := tc.agentProc.processMon.MemoryInfo()
@@ -297,7 +297,7 @@ func (tc *TestCase) WaitForN(cond func() bool, duration time.Duration, errMsg ..
 
 // WaitFor is like WaitForN but with a fixed duration of 10 seconds
 func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool {
-	return tc.WaitForN(cond, time.Second * 10, errMsg...)
+	return tc.WaitForN(cond, time.Second*10, errMsg...)
 }
 
 func (tc *TestCase) indicateError(err error) {

diff --git a/testbed/tests/perf_test.go b/testbed/tests/perf_test.go
@@ -100,55 +100,25 @@ func TestNoBackend10kSPS(t *testing.T) {
 	tc.Sleep(10 * time.Second)
 }
 
-func Test1000SPSWithAttributes(t *testing.T) {
-
-	tests := []struct {
-		attrCount    int
-		attrSizeByte int
-		expectedCPU  uint32
-	}{
-		// No attributes.
-		{
-			attrCount:    0,
-			attrSizeByte: 0,
-			expectedCPU:  20,
-		},
-
-		// We generate 10 attributes each with average key length of 100 bytes and
-		// average value length of 50 bytes so total size of attributes values is
-		// 15000 bytes.
-		{
-			attrCount:    100,
-			attrSizeByte: 50,
-			expectedCPU:  120,
-		},
-
-		// Approx 10 KiB attributes.
-		{
-			attrCount:    10,
-			attrSizeByte: 1000,
-			expectedCPU:  100,
-		},
-
-		// Approx 100 KiB attributes.
-		{
-			attrCount:    20,
-			attrSizeByte: 5000,
-			expectedCPU:  250,
-		},
-	}
+type testCase struct {
+	attrCount    int
+	attrSizeByte int
+	expectedMaxCPU  uint32
+	expectedMaxRAM uint32
+}
 
+func test1000SPSWithAttributes(t *testing.T, args []string, tests []testCase) {
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%d*%dbytes", test.attrCount, test.attrSizeByte), func(t *testing.T) {
 
 			tc := testbed.NewTestCase(t)
 			defer tc.Stop()
 
-			tc.SetExpectedMaxCPU(test.expectedCPU)
-			tc.SetExpectedMaxRAM(100)
+			tc.SetExpectedMaxCPU(test.expectedMaxCPU)
+			tc.SetExpectedMaxRAM(test.expectedMaxRAM)
 
 			tc.StartBackend(testbed.BackendOC)
-			tc.StartAgent()
+			tc.StartAgent(args...)
 
 			options := testbed.LoadOptions{SpansPerSecond: 1000}
 			options.Attributes = make(map[string]interface{})
@@ -172,3 +142,73 @@ func Test1000SPSWithAttributes(t *testing.T) {
 		})
 	}
 }
+
+
+func Test1000SPSWithAttributes(t *testing.T) {
+	test1000SPSWithAttributes(t, []string{}, []testCase{
+		// No attributes.
+		{
+			attrCount:    0,
+			attrSizeByte: 0,
+			expectedMaxCPU:  30,
+			expectedMaxRAM:  100,
+		},
+
+		// We generate 10 attributes each with average key length of 100 bytes and
+		// average value length of 50 bytes so total size of attributes values is
+		// 15000 bytes.
+		{
+			attrCount:    100,
+			attrSizeByte: 50,
+			expectedMaxCPU:  120,
+			expectedMaxRAM:  100,
+		},
+
+		// Approx 10 KiB attributes.
+		{
+			attrCount:    10,
+			attrSizeByte: 1000,
+			expectedMaxCPU:  100,
+			expectedMaxRAM:  100,
+		},
+
+		// Approx 100 KiB attributes.
+		{
+			attrCount:    20,
+			attrSizeByte: 5000,
+			expectedMaxCPU:  250,
+			expectedMaxRAM:  100,
+		},
+	})
+}
+
+func TestBallast1000SPSWithAttributes(t *testing.T) {
+	args := []string{"--mem-ballast-size-mib", "1000"}
+	test1000SPSWithAttributes(t, args, []testCase{
+		// No attributes.
+		{
+			attrCount:    0,
+			attrSizeByte: 0,
+			expectedMaxCPU:  30,
+			expectedMaxRAM:  2000,
+		},
+		{
+			attrCount:    100,
+			attrSizeByte: 50,
+			expectedMaxCPU:  80,
+			expectedMaxRAM:  2000,
+		},
+		{
+			attrCount:    10,
+			attrSizeByte: 1000,
+			expectedMaxCPU:  80,
+			expectedMaxRAM:  2000,
+		},
+		{
+			attrCount:    20,
+			attrSizeByte: 5000,
+			expectedMaxCPU:  120,
+			expectedMaxRAM:  2000,
+		},
+	})
+}
diff --git a/testbed/tests/results/BASELINE.md b/testbed/tests/results/BASELINE.md
@@ -1,14 +1,18 @@
 # Test Results
-Started: Tue, 25 Jun 2019 10:31:56 -0400
+Started: Wed, 03 Jul 2019 05:42:19 +0530
 
-Test|Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Spans|Received Spans
-----|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:
-TestIdleMode|Passed|10s|0.3|1.0|25|34|0|0
-Test10kSPS|Passed|17s|91.4|95.7|35|43|149610|149610
-TestNoBackend10kSPS|Passed|10s|59.1|59.3|29|40|99910|0
-Test1000SPSWithAttributes/0*0bytes|Passed|12s|18.7|19.3|30|41|10000|10000
-Test1000SPSWithAttributes/100*50bytes|Passed|12s|70.2|71.0|32|45|10000|10000
-Test1000SPSWithAttributes/10*1000bytes|Passed|12s|68.0|74.3|32|44|10000|10000
-Test1000SPSWithAttributes/20*5000bytes|Passed|12s|158.2|173.7|39|55|10000|10000
+Test                                    |Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Spans|Received Spans
+----------------------------------------|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:
+TestIdleMode                            |PASS  |     10s|     0.3|     1.0|         25|         34|         0|             0
+Test10kSPS                              |PASS  |     16s|    72.6|    92.0|         34|         41|    144520|        144520
+TestNoBackend10kSPS                     |PASS  |     10s|    75.6|    78.3|         29|         40|     97460|             0
+Test1000SPSWithAttributes/0*0bytes      |PASS  |     11s|    22.3|    24.0|         31|         42|     10000|         10000
+Test1000SPSWithAttributes/100*50bytes   |PASS  |     11s|    92.3|   106.3|         33|         46|      9990|          9990
+Test1000SPSWithAttributes/10*1000bytes  |PASS  |     11s|    86.2|    93.3|         34|         46|     10000|         10000
+Test1000SPSWithAttributes/20*5000bytes  |PASS  |     11s|   126.3|   130.0|         47|         64|      9990|          9990
+TestBallast1000SPSWithAttributes/0*0bytes|PASS  |     11s|    20.4|    21.7|         71|        109|     10000|         10000
+TestBallast1000SPSWithAttributes/100*50bytes|PASS  |     11s|    64.4|    68.0|        492|        845|      9950|          9950
+TestBallast1000SPSWithAttributes/10*1000bytes|PASS  |     11s|    55.1|    59.7|        350|        575|      9990|          9990
+TestBallast1000SPSWithAttributes/20*5000bytes|PASS  |     11s|    67.8|    69.3|        786|       1081|      9950|          9950
 
-Total duration: 87s
+Total duration: 127s