Basic Async Unary implementation of QPS benchmark

Performance/QPSBenchmark/.gitignore

@@ -0,0 +1 @@
+Package.resolved

Performance/QPSBenchmark/Makefile

@@ -0,0 +1,63 @@
+# Which Swift to use.
+SWIFT:=swift
+# Where products will be built; this is the SPM default.
+GRPC_SWIFT_PATH:=../..
+SWIFT_BUILD_PATH:=./.build
+SWIFT_BUILD_CONFIGURATION=release
+SWIFT_FLAGS=--build-path=${SWIFT_BUILD_PATH} --configuration=${SWIFT_BUILD_CONFIGURATION}
+
+# protoc plugins.
+PROTOC_GEN_SWIFT=${GRPC_SWIFT_PATH}/${SWIFT_BUILD_PATH}/release/protoc-gen-swift
+PROTOC_GEN_GRPC_SWIFT=${GRPC_SWIFT_PATH}/${SWIFT_BUILD_PATH}/release/protoc-gen-grpc-swift
+
+SWIFT_BUILD:=${SWIFT} build ${SWIFT_FLAGS}
+SWIFT_BUILD_RELEASE:=${SWIFT} build ${SWIFT_FLAGS_RELEASE}
+
+### Package and plugin build targets ###########################################
+
+all:
+	${SWIFT_BUILD}
+
+${PROTOC_GEN_SWIFT}: ${GRPC_SWIFT_PATH}/Package.resolved
+	${SWIFT_BUILD_RELEASE} --product protoc-gen-swift --package-path ${GRPC_SWIFT_PATH}
+
+${PROTOC_GEN_GRPC_SWIFT}: ${GRPC_SWIFT_PATH}/Sources/protoc-gen-grpc-swift/*.swift
+	${SWIFT_BUILD_RELEASE} --product protoc-gen-grpc-swift --package-path ${GRPC_SWIFT_PATH}
+
+### Protobuf Generation ########################################################
+
+%.pb.swift: %.proto ${PROTOC_GEN_SWIFT}
+	protoc $< \
+		--proto_path=$(dir $<) \
+		--plugin=${PROTOC_GEN_SWIFT} \
+		--swift_opt=Visibility=Public \
+		--swift_out=$(dir $<)
+
+%.grpc.swift: %.proto ${PROTOC_GEN_GRPC_SWIFT}
+	protoc $< \
+		--proto_path=$(dir $<) \
+		--plugin=${PROTOC_GEN_GRPC_SWIFT} \
+		--grpc-swift_opt=Visibility=Public \
+		--grpc-swift_out=$(dir $<)
+
+QBS_PROTO=Sources/QPSBenchmark/Model/benchmark_service.proto \
+          Sources/QPSBenchmark/Model/payloads.proto \
+          Sources/QPSBenchmark/Model/control.proto \
+          Sources/QPSBenchmark/Model/stats.proto \
+          Sources/QPSBenchmark/Model/core_stats.proto \
+          Sources/QPSBenchmark/Model/worker_service.proto \
+          Sources/QPSBenchmark/Model/messages.proto
+QBS_PB=$(QBS_PROTO:.proto=.pb.swift)
+QBS_GRPC=$(QBS_PROTO:.proto=.grpc.swift)
+
+# Generate the protobufs for the QPS benchmarking worker.
+.PHONY:
+generate-qps-worker: ${QBS_PB} ${QBS_GRPC}
+
+### Misc. ######################################################################
+
+.PHONY:
+clean:
+	-rm -rf Packages
+	-rm -rf ${SWIFT_BUILD_PATH}
+	-rm -f Package.resolved

Performance/QPSBenchmark/Package.swift

@@ -0,0 +1,55 @@
+// swift-tools-version:5.0
+/*
+ * Copyright 2020, gRPC Authors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PackageDescription
+
+let package = Package(
+  name: "QPSBenchmark",
+  products: [
+    .executable(name: "QPSBenchmark", targets: ["QPSBenchmark"]),
+  ],
+
+  dependencies: [
+    .package(path: "../../"),
+    .package(url: "https://github.com/apple/swift-nio.git", from: "2.22.0"),
+    .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"),
+    .package(url: "https://github.com/apple/swift-argument-parser.git", from: "0.3.0"),
+    .package(
+      url: "https://github.com/swift-server/swift-service-lifecycle.git",
+      from: "1.0.0-alpha"
+    ),
+    .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.9.0"),
+  ],
+  targets: [
+    .target(name: "QPSBenchmark", dependencies: [
+      "GRPC",
+      "NIO",
+      "ArgumentParser",
+      "Logging",
+      "Lifecycle",
+      "SwiftProtobuf",
+      "BenchmarkUtils",
+    ]),
+    .target(name: "BenchmarkUtils", dependencies: [
+      "GRPC",
+    ]),
+    .testTarget(name: "BenchmarkUtilsTests", dependencies: [
+      "GRPC",
+      "BenchmarkUtils",
+    ]),
+  ]
+)

Performance/QPSBenchmark/README.md

@@ -0,0 +1,28 @@
+#  QPS Benchmark worker
+
+An implementation of the QPS worker for benchmarking described in the  
+[gRPC benchmarking guide](https://grpc.io/docs/guides/benchmarking/)
+
+## Building
+To rebuild the proto files run `make generate-qps-worker`.
+
+The benchmarks can be built in the usual SPM way but release mode is strongly recommended - `swift build -c release`
+
+## Running the benchmarks
+
+To date the changes to gRPC to run the tests automatically have not been pushed upstream.
+
+You can easily run the tests locally using the C++ driver program from gRPC - note this is built as a side effect 
+of running the C++ tests which can be done in a gRPC checkout with 
+`./tools/run_tests/run_performance_tests.py -l c++ -r cpp_protobuf_async_unary_qps_unconstrained_insecure`
+
+For an example of running a benchmarking tests proceed as follows
+1. Open a terminal window and run the QPSBenchmark - `swift run -c release QPSBenchmark --driver_port 10400`.  
+This will become the server when instructed by the driver.
+2. Open another terminal window and run QPSBenchmark - `swift run -c release QPSBenchmark --driver_port 10410`.
+This will become the client when instructed by the driver.
+3. Use the driver to control the test.  In your checkout of [gRPC](https://github.com/grpc/grpc) 
+configure the environment with `export QPS_WORKERS="localhost:10400,localhost:10410"` then run
+`cmake/build/qps_json_driver '--scenarios_json={"scenarios": [{"name": "swift_protobuf_async_unary_qps_unconstrained_insecure", "warmup_seconds": 5, "benchmark_seconds": 30, "num_servers": 1, "server_config": {"async_server_threads": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "server_type": "ASYNC_SERVER", "security_params": null, "threads_per_cq": 0, "server_processes": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0, "load_params": {"closed_loop": {}}, "client_type": "ASYNC_CLIENT", "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}, "client_processes": 0}, "num_clients": 0}]}' --scenario_result_file=scenario_result.json`
+This will run a test of asynchronous unary client and server, using all the cores on the machine.  
+64 channels each with 100 outstanding requests.

Performance/QPSBenchmark/Sources/BenchmarkUtils/Histogram.swift

@@ -0,0 +1,122 @@
+/*
+ * Copyright 2020, gRPC Authors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+struct HistorgramShapeMismatch: Error {}
+
+/// Histograms are stored with exponentially increasing bucket sizes.
+/// The first bucket is [0, `multiplier`) where `multiplier` = 1 + resolution
+/// Bucket n (n>=1) contains [`multiplier`**n, `multiplier`**(n+1))
+/// There are sufficient buckets to reach max_bucket_start
+public struct Histogram {
+  public private(set) var sum: Double
+  public private(set) var sumOfSquares: Double
+  public private(set) var countOfValuesSeen: Double
+  private var multiplier: Double
+  private var oneOnLogMultiplier: Double
+  public private(set) var minSeen: Double
+  public private(set) var maxSeen: Double
+  private var maxPossible: Double
+  public private(set) var buckets: [UInt32]
+
+  /// Initialise a histogram.
+  /// - parameters:
+  ///     - resolution: Defines the width of the buckets - see the description of this structure.
+  ///     - maxBucketStart: Defines the start of the greatest valued bucket.
+  public init(resolution: Double = 0.01, maxBucketStart: Double = 60e9) {
+    precondition(resolution > 0.0)
+    precondition(maxBucketStart > resolution)
+    self.sum = 0.0
+    self.sumOfSquares = 0.0
+    self.multiplier = 1.0 + resolution
+    self.oneOnLogMultiplier = 1.0 / log(1.0 + resolution)
+    self.maxPossible = maxBucketStart
+    self.countOfValuesSeen = 0.0
+    self.minSeen = maxBucketStart
+    self.maxSeen = 0.0
+    let numBuckets = Histogram.bucketForUnchecked(
+      value: maxBucketStart,
+      oneOnLogMultiplier: self.oneOnLogMultiplier
+    ) + 1
+    precondition(numBuckets > 1)
+    precondition(numBuckets < 100_000_000)
+    self.buckets = .init(repeating: 0, count: numBuckets)
+  }
+
+  /// Determine a bucket index given a value - does no bounds checking
+  private static func bucketForUnchecked(value: Double, oneOnLogMultiplier: Double) -> Int {
+    return Int(log(value) * oneOnLogMultiplier)
+  }
+
+  private func bucketFor(value: Double) -> Int {
+    let bucket = Histogram.bucketForUnchecked(
+      value: self.clamp(value: value),
+      oneOnLogMultiplier: self.oneOnLogMultiplier
+    )
+    assert(bucket < self.buckets.count)
+    assert(bucket >= 0)
+    return bucket
+  }
+
+  /// Force a value to be within the bounds of 0 and `self.maxPossible`
+  /// - parameters:
+  ///     - value: The value to force within bounds
+  /// - returns: The value forced into the bounds for buckets.
+  private func clamp(value: Double) -> Double {
+    return min(self.maxPossible, max(0, value))
+  }
+
+  /// Add a value to this histogram, updating buckets and stats
+  /// - parameters:
+  ///     - value: The value to add.
+  public mutating func add(value: Double) {
+    self.sum += value
+    self.sumOfSquares += value * value
+    self.countOfValuesSeen += 1
+    if value < self.minSeen {
+      self.minSeen = value
+    }
+    if value > self.maxSeen {
+      self.maxSeen = value
+    }
+    self.buckets[self.bucketFor(value: value)] += 1
+  }
+
+  /// Merge two histograms together updating `self`
+  /// - parameters:
+  ///    - source: the other histogram to merge into this.
+  public mutating func merge(source: Histogram) throws {
+    guard (self.buckets.count == source.buckets.count) ||
+      (self.multiplier == source.multiplier) else {
+      // Fail because these histograms don't match.
+      throw HistorgramShapeMismatch()
+    }
+
+    self.sum += source.sum
+    self.sumOfSquares += source.sumOfSquares
+    self.countOfValuesSeen += source.countOfValuesSeen
+    if source.minSeen < self.minSeen {
+      self.minSeen = source.minSeen
+    }
+    if source.maxSeen > self.maxSeen {
+      self.maxSeen = source.maxSeen
+    }
+    for bucket in 0 ..< self.buckets.count {
+      self.buckets[bucket] += source.buckets[bucket]
+    }
+  }
+}

Performance/QPSBenchmark/Sources/BenchmarkUtils/StatusCounts.swift

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2020, gRPC Authors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import GRPC
+
+/// Count the number seen of each status code.
+public struct StatusCounts {
+  public private(set) var counts: [Int: Int64] = [:]
+
+  public init() {}
+
+  /// Add one to the count of this sort of status code.
+  /// - parameters:
+  ///     - status: The code to count.
+  public mutating func add(status: GRPCStatus.Code) {
+    // Only record failures
+    if status != .ok {
+      if let previousCount = self.counts[status.rawValue] {
+        self.counts[status.rawValue] = previousCount + 1
+      } else {
+        self.counts[status.rawValue] = 1
+      }
+    }
+  }
+
+  /// Merge another set of counts into this one.
+  /// - parameters:
+  ///     - source: The other set of counts to merge into this.
+  public mutating func merge(source: StatusCounts) {
+    for sourceCount in source.counts {
+      if let existingCount = self.counts[sourceCount.key] {
+        self.counts[sourceCount.key] = existingCount + sourceCount.value
+      } else {
+        self.counts[sourceCount.key] = sourceCount.value
+      }
+    }
+  }
+}