diff --git a/.travis-script.sh b/.travis-script.sh index b59fa7e74..561b8ae62 100755 --- a/.travis-script.sh +++ b/.travis-script.sh @@ -56,6 +56,15 @@ make_test() { echo -en 'travis_fold:end:make.test\\r' } +run_allocation_tests() { + echo -en 'travis_fold:start:allocation_tests\\r' + + info "Running allocation counter tests" + ./Performance/allocations/test-allocation-counts.sh + + echo -en 'travis_fold:end:allocation_tests\\r' +} + make_test_plugin() { echo -en 'travis_fold:start:make.test_plugin\\r' info "Validating protoc plugins on the Echo service" @@ -152,10 +161,11 @@ run_interop_reconnect_test() { } just_sanity=false +allocation_tests=false just_interop_tests=false tsan=false -while getopts "sit" optname; do +while getopts "sita" optname; do case $optname in s) just_sanity=true @@ -166,6 +176,9 @@ while getopts "sit" optname; do t) tsan=true ;; + a) + allocation_tests=true + ;; \?) echo "Uknown option $optname" exit 2 @@ -183,6 +196,9 @@ elif $just_interop_tests; then else make_all make_test $tsan + if $allocation_tests; then + run_allocation_tests + fi make_test_plugin make_project fi diff --git a/.travis.yml b/.travis.yml index 98434b849..a2eafc8d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ jobs: dist: bionic install: ./.travis-install.sh -f # install swiftformat script: ./.travis-script.sh -s # just sanity - env: SWIFT_VERSION=5.3.1 + env: SWIFT_VERSION=5.3.3 # Tests for each PR. - &tests stage: "Test" @@ -34,12 +34,20 @@ jobs: os: linux dist: bionic install: ./.travis-install.sh -p # install protoc - script: ./.travis-script.sh -t # with tsan - env: SWIFT_VERSION=5.3.1 + script: ./.travis-script.sh -t -a # tests with tsan, run allocation tests + env: + - SWIFT_VERSION=5.3.3 + - MAX_ALLOCS_ALLOWED_embedded_server_bidi_1k_rpcs_10_small_requests=113000 + - MAX_ALLOCS_ALLOWED_embedded_server_bidi_1k_rpcs_1_small_request=68000 + - MAX_ALLOCS_ALLOWED_embedded_server_unary_1k_rpcs_1_small_request=64000 - <<: *tests name: "Unit Tests: Ubuntu 18.04 (Swift 5.2)" - script: ./.travis-script.sh - env: SWIFT_VERSION=5.2.5 + script: ./.travis-script.sh -a # run allocation tests + env: + - SWIFT_VERSION=5.2.5 + - MAX_ALLOCS_ALLOWED_embedded_server_bidi_1k_rpcs_10_small_requests=113000 + - MAX_ALLOCS_ALLOWED_embedded_server_bidi_1k_rpcs_1_small_request=68000 + - MAX_ALLOCS_ALLOWED_embedded_server_unary_1k_rpcs_1_small_request=64000 - <<: *tests name: "Unit Tests: Xcode 12.2" os: osx @@ -57,7 +65,7 @@ jobs: dist: bionic install: ./.travis-install.sh -p -i # install protoc and interop server script: ./.travis-script.sh -i # interop tests - env: SWIFT_VERSION=5.3.1 + env: SWIFT_VERSION=5.3.3 - <<: *interop_tests name: "Interoperability Tests: Ubuntu 18.04 (Swift 5.2)" env: SWIFT_VERSION=5.2.5 diff --git a/NOTICES.txt b/NOTICES.txt new file mode 100644 index 000000000..f1ff7bbab --- /dev/null +++ b/NOTICES.txt @@ -0,0 +1,27 @@ + The gRPC Swift Project + ====================== + +Copyright 2021 The gRPC Swift Project + +The gRPC Swift project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +------------------------------------------------------------------------------- + +This product uses scripts derived from SwiftNIO's integration testing +framework: 'test_01_allocation_counts.sh', 'run-nio-alloc-counter-tests.sh' and +'test_functions.sh'. + + * LICENSE (Apache License 2.0): + * https://github.com/apple/swift-nio/blob/main/LICENSE.txt + * HOMEPAGE: + * https://github.com/apple/swift-nio diff --git a/Package.swift b/Package.swift index 40979e494..b1f403ab4 100644 --- a/Package.swift +++ b/Package.swift @@ -140,7 +140,6 @@ let package = Package( name: "GRPCPerformanceTests", dependencies: [ .target(name: "GRPC"), - .target(name: "EchoModel"), .product(name: "NIO", package: "swift-nio"), .product(name: "ArgumentParser", package: "swift-argument-parser"), ] diff --git a/Performance/allocations/test-allocation-counts.sh b/Performance/allocations/test-allocation-counts.sh new file mode 100755 index 000000000..dcdc894c6 --- /dev/null +++ b/Performance/allocations/test-allocation-counts.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Copyright 2021, gRPC Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script was adapted from SwiftNIO's test_01_allocation_counts.sh. The +# license for the original work is reproduced below. See NOTICES.txt for more. + +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -eu +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +tmp="/tmp" + +source "$here/test-utils.sh" + +all_tests=() +for file in "$here/tests/"test_*.swift; do + # Extract the "TESTNAME" from "test_TESTNAME.swift" + test_name=$(basename "$file") + test_name=${test_name#test_*} + test_name=${test_name%*.swift} + all_tests+=( "$test_name" ) +done + +# Run all the tests. +"$here/tests/run-allocation-counter-tests.sh" -t "$tmp" > "$tmp/output" + +# Dump some output from each, check for allocations. +for test in "${all_tests[@]}"; do + cat "$tmp/output" # helps debugging + + while read -r test_case; do + test_case=${test_case#test_*} + total_allocations=$(grep "^test_$test_case.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') + not_freed_allocations=$(grep "^test_$test_case.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g') + max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test_case" + + info "$test_case: allocations not freed: $not_freed_allocations" + info "$test_case: total number of mallocs: $total_allocations" + + assert_less_than "$not_freed_allocations" 5 # allow some slack + assert_greater_than "$not_freed_allocations" -5 # allow some slack + if [[ -z "${!max_allowed_env_name+x}" ]]; then + if [[ -z "${!max_allowed_env_name+x}" ]]; then + warn "no reference number of allocations set (set to \$$max_allowed_env_name)" + warn "to set current number:" + warn " export $max_allowed_env_name=$total_allocations" + fi + else + max_allowed=${!max_allowed_env_name} + assert_less_than_or_equal "$total_allocations" "$max_allowed" + assert_greater_than "$total_allocations" "$(( max_allowed - 1000))" + fi + done < <(grep "^test_$test[^\W]*.total_allocations:" "$tmp/output" | cut -d: -f1 | cut -d. -f1 | sort | uniq) +done diff --git a/Performance/allocations/test-utils.sh b/Performance/allocations/test-utils.sh new file mode 100755 index 000000000..70688f2cf --- /dev/null +++ b/Performance/allocations/test-utils.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright 2021, gRPC Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script contains part of SwiftNIO's test_functions.sh script. The license +# for the original work is reproduced below. See NOTICES.txt for more. + +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +function fail() { + echo >&2 "FAILURE: $*" + false +} + +function assert_less_than() { + if [[ ! "$1" -lt "$2" ]]; then + fail "assertion '$1' < '$2' failed" + fi +} + +function assert_less_than_or_equal() { + if [[ ! "$1" -le "$2" ]]; then + fail "assertion '$1' <= '$2' failed" + fi +} + +function assert_greater_than() { + if [[ ! "$1" -gt "$2" ]]; then + fail "assertion '$1' > '$2' failed" + fi +} + +g_has_previously_infoed=false + +function info() { + if ! $g_has_previously_infoed; then + echo || true # echo an extra newline so it looks better + g_has_previously_infoed=true + fi + echo "info: $*" || true +} + +function warn() { + echo "warning: $*" +} diff --git a/Performance/allocations/tests/run-allocation-counter-tests.sh b/Performance/allocations/tests/run-allocation-counter-tests.sh new file mode 100755 index 000000000..a37a1985d --- /dev/null +++ b/Performance/allocations/tests/run-allocation-counter-tests.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Copyright 2021, gRPC Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script was adapted from SwiftNIO's 'run-nio-alloc-counter-tests.sh' +# script. The license for the original work is reproduced below. See NOTICES.txt +# for more. + +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -eu +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +tmp_dir="/tmp" + +while getopts "t:" opt; do + case "$opt" in + t) + tmp_dir="$OPTARG" + ;; + *) + exit 1 + ;; + esac +done + +nio_checkout=$(mktemp -d "$tmp_dir/.swift-nio_XXXXXX") +( +cd "$nio_checkout" +git clone --depth 1 https://github.com/apple/swift-nio +) + +shift $((OPTIND-1)) + +tests_to_run=("$here"/test_*.swift) + +if [[ $# -gt 0 ]]; then + tests_to_run=("$@") +fi + +# We symlink in a bunch of components from the GRPCPerformanceTests target to +# avoid duplicating a bunch of code. +"$nio_checkout/swift-nio/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh" \ + -p "$here/../../.." \ + -m GRPC \ + -t "$tmp_dir" \ + -s "$here/shared/Benchmark.swift" \ + -s "$here/shared/echo.pb.swift" \ + -s "$here/shared/echo.grpc.swift" \ + -s "$here/shared/MinimalEchoProvider.swift" \ + -s "$here/shared/EmbeddedServer.swift" \ + "${tests_to_run[@]}" diff --git a/Performance/allocations/tests/shared/Benchmark.swift b/Performance/allocations/tests/shared/Benchmark.swift new file mode 120000 index 000000000..56c05abe6 --- /dev/null +++ b/Performance/allocations/tests/shared/Benchmark.swift @@ -0,0 +1 @@ +../../../../Sources/GRPCPerformanceTests/Benchmark.swift \ No newline at end of file diff --git a/Performance/allocations/tests/shared/EmbeddedServer.swift b/Performance/allocations/tests/shared/EmbeddedServer.swift new file mode 120000 index 000000000..9c1d98aec --- /dev/null +++ b/Performance/allocations/tests/shared/EmbeddedServer.swift @@ -0,0 +1 @@ +../../../../Sources/GRPCPerformanceTests/Benchmarks/EmbeddedServer.swift \ No newline at end of file diff --git a/Performance/allocations/tests/shared/MinimalEchoProvider.swift b/Performance/allocations/tests/shared/MinimalEchoProvider.swift new file mode 120000 index 000000000..0ad07f378 --- /dev/null +++ b/Performance/allocations/tests/shared/MinimalEchoProvider.swift @@ -0,0 +1 @@ +../../../../Sources/GRPCPerformanceTests/Benchmarks/MinimalEchoProvider.swift \ No newline at end of file diff --git a/Performance/allocations/tests/shared/echo.grpc.swift b/Performance/allocations/tests/shared/echo.grpc.swift new file mode 120000 index 000000000..2c0efcf5c --- /dev/null +++ b/Performance/allocations/tests/shared/echo.grpc.swift @@ -0,0 +1 @@ +../../../../Sources/GRPCPerformanceTests/Benchmarks/echo.grpc.swift \ No newline at end of file diff --git a/Performance/allocations/tests/shared/echo.pb.swift b/Performance/allocations/tests/shared/echo.pb.swift new file mode 120000 index 000000000..64ef8232c --- /dev/null +++ b/Performance/allocations/tests/shared/echo.pb.swift @@ -0,0 +1 @@ +../../../../Sources/GRPCPerformanceTests/Benchmarks/echo.pb.swift \ No newline at end of file diff --git a/Performance/allocations/tests/test_embedded_server_bidi_1k_rpcs_10_small_requests.swift b/Performance/allocations/tests/test_embedded_server_bidi_1k_rpcs_10_small_requests.swift new file mode 100644 index 000000000..cca8d4e24 --- /dev/null +++ b/Performance/allocations/tests/test_embedded_server_bidi_1k_rpcs_10_small_requests.swift @@ -0,0 +1,24 @@ +/* + * Copyright 2021, gRPC Authors All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +func run(identifier: String) { + measure(identifier: identifier) { + let benchmark = EmbeddedServerChildChannelBenchmark( + mode: .bidirectional(rpcs: 1000, requestsPerRPC: 10), + text: "" + ) + return try! benchmark.runOnce() + } +} diff --git a/Performance/allocations/tests/test_embedded_server_bidi_1k_rpcs_1_small_request.swift b/Performance/allocations/tests/test_embedded_server_bidi_1k_rpcs_1_small_request.swift new file mode 100644 index 000000000..98ba38f2f --- /dev/null +++ b/Performance/allocations/tests/test_embedded_server_bidi_1k_rpcs_1_small_request.swift @@ -0,0 +1,24 @@ +/* + * Copyright 2021, gRPC Authors All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +func run(identifier: String) { + measure(identifier: identifier) { + let benchmark = EmbeddedServerChildChannelBenchmark( + mode: .bidirectional(rpcs: 1000, requestsPerRPC: 1), + text: "" + ) + return try! benchmark.runOnce() + } +} diff --git a/Performance/allocations/tests/test_embedded_server_unary_1k_rpcs_1_small_request.swift b/Performance/allocations/tests/test_embedded_server_unary_1k_rpcs_1_small_request.swift new file mode 100644 index 000000000..c9a80e157 --- /dev/null +++ b/Performance/allocations/tests/test_embedded_server_unary_1k_rpcs_1_small_request.swift @@ -0,0 +1,21 @@ +/* + * Copyright 2021, gRPC Authors All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +func run(identifier: String) { + measure(identifier: identifier) { + let benchmark = EmbeddedServerChildChannelBenchmark(mode: .unary(rpcs: 1000), text: "") + return try! benchmark.runOnce() + } +} diff --git a/Sources/GRPCPerformanceTests/Benchmark.swift b/Sources/GRPCPerformanceTests/Benchmark.swift index 0aaa49be2..47cdedc6c 100644 --- a/Sources/GRPCPerformanceTests/Benchmark.swift +++ b/Sources/GRPCPerformanceTests/Benchmark.swift @@ -13,85 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import Dispatch protocol Benchmark: AnyObject { func setUp() throws func tearDown() throws - func run() throws + func run() throws -> Int } -/// The results of a benchmark. -struct BenchmarkResults { - /// The description of the benchmark. - var desc: String - - /// The duration of each run of the benchmark in milliseconds. - var milliseconds: [UInt64] -} - -extension BenchmarkResults: CustomStringConvertible { - var description: String { - return "\(self.desc): \(self.milliseconds.map(String.init).joined(separator: ","))" - } -} - -/// Runs the benchmark and prints the duration in milliseconds for each run. -/// -/// - Parameters: -/// - description: A description of the benchmark. -/// - benchmark: The benchmark which should be run. -/// - spec: The specification for the test run. -func measureAndPrint(description: String, benchmark: Benchmark, spec: TestSpec) { - switch spec.action { - case .list: - print(description) - case let .run(filter): - guard filter.shouldRun(description) else { - return - } - #if CACHEGRIND - _ = measure(description, benchmark: benchmark, repeats: 1) - #else - print(measure(description, benchmark: benchmark, repeats: spec.repeats)) - #endif - } -} - -/// Runs the given benchmark multiple times, recording the wall time for each iteration. -/// -/// - Parameters: -/// - description: A description of the benchmark. -/// - benchmark: The benchmark to run. -/// - repeats: the number of times to run the benchmark. -func measure(_ description: String, benchmark: Benchmark, repeats: Int) -> BenchmarkResults { - var milliseconds: [UInt64] = [] - for _ in 0 ..< repeats { - do { - try benchmark.setUp() - - #if !CACHEGRIND - let start = DispatchTime.now().uptimeNanoseconds - #endif - try benchmark.run() - - #if !CACHEGRIND - let end = DispatchTime.now().uptimeNanoseconds - - milliseconds.append((end - start) / 1_000_000) - #endif - } catch { - // If tearDown fails now then there's not a lot we can do! - try? benchmark.tearDown() - return BenchmarkResults(desc: description, milliseconds: []) - } - - do { - try benchmark.tearDown() - } catch { - return BenchmarkResults(desc: description, milliseconds: []) - } +extension Benchmark { + func runOnce() throws -> Int { + try self.setUp() + let result = try self.run() + try self.tearDown() + return result } - - return BenchmarkResults(desc: description, milliseconds: milliseconds) } diff --git a/Sources/GRPCPerformanceTests/BenchmarkUtils.swift b/Sources/GRPCPerformanceTests/BenchmarkUtils.swift new file mode 100644 index 000000000..b88bc16d6 --- /dev/null +++ b/Sources/GRPCPerformanceTests/BenchmarkUtils.swift @@ -0,0 +1,91 @@ +/* + * Copyright 2021, gRPC Authors All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Dispatch + +/// The results of a benchmark. +struct BenchmarkResults { + /// The description of the benchmark. + var desc: String + + /// The duration of each run of the benchmark in milliseconds. + var milliseconds: [UInt64] +} + +extension BenchmarkResults: CustomStringConvertible { + var description: String { + return "\(self.desc): \(self.milliseconds.map(String.init).joined(separator: ","))" + } +} + +/// Runs the benchmark and prints the duration in milliseconds for each run. +/// +/// - Parameters: +/// - description: A description of the benchmark. +/// - benchmark: The benchmark which should be run. +/// - spec: The specification for the test run. +func measureAndPrint(description: String, benchmark: Benchmark, spec: TestSpec) { + switch spec.action { + case .list: + print(description) + case let .run(filter): + guard filter.shouldRun(description) else { + return + } + #if CACHEGRIND + _ = measure(description, benchmark: benchmark, repeats: 1) + #else + print(measure(description, benchmark: benchmark, repeats: spec.repeats)) + #endif + } +} + +/// Runs the given benchmark multiple times, recording the wall time for each iteration. +/// +/// - Parameters: +/// - description: A description of the benchmark. +/// - benchmark: The benchmark to run. +/// - repeats: the number of times to run the benchmark. +func measure(_ description: String, benchmark: Benchmark, repeats: Int) -> BenchmarkResults { + var milliseconds: [UInt64] = [] + for _ in 0 ..< repeats { + do { + try benchmark.setUp() + + #if !CACHEGRIND + let start = DispatchTime.now().uptimeNanoseconds + #endif + _ = try benchmark.run() + + #if !CACHEGRIND + let end = DispatchTime.now().uptimeNanoseconds + + milliseconds.append((end - start) / 1_000_000) + #endif + } catch { + // If tearDown fails now then there's not a lot we can do! + try? benchmark.tearDown() + return BenchmarkResults(desc: description, milliseconds: []) + } + + do { + try benchmark.tearDown() + } catch { + return BenchmarkResults(desc: description, milliseconds: []) + } + } + + return BenchmarkResults(desc: description, milliseconds: milliseconds) +} diff --git a/Sources/GRPCPerformanceTests/Benchmarks/EmbeddedClientThroughput.swift b/Sources/GRPCPerformanceTests/Benchmarks/EmbeddedClientThroughput.swift index 523229685..0688fbe4b 100644 --- a/Sources/GRPCPerformanceTests/Benchmarks/EmbeddedClientThroughput.swift +++ b/Sources/GRPCPerformanceTests/Benchmarks/EmbeddedClientThroughput.swift @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import EchoModel import struct Foundation.Data import GRPC import Logging @@ -77,7 +76,9 @@ class EmbeddedClientThroughput: Benchmark { func tearDown() throws {} - func run() throws { + func run() throws -> Int { + var messages = 0 + for _ in 0 ..< self.requestCount { let channel = EmbeddedChannel() @@ -93,12 +94,11 @@ class EmbeddedClientThroughput: Benchmark { // Write the request parts. try channel.writeOutbound(_GRPCClientRequestPart.head(self.requestHead)) - try channel - .writeOutbound( - _GRPCClientRequestPart - .message(.init(self.request, compressed: false)) - ) + try channel.writeOutbound( + _GRPCClientRequestPart.message(.init(self.request, compressed: false)) + ) try channel.writeOutbound(_GRPCClientRequestPart.end) + messages += 1 // Read out the request frames. var requestFrames = 0 @@ -140,5 +140,7 @@ class EmbeddedClientThroughput: Benchmark { precondition(responseParts == 4, "received \(responseParts) response parts") } + + return messages } } diff --git a/Sources/GRPCPerformanceTests/Benchmarks/EmbeddedServer.swift b/Sources/GRPCPerformanceTests/Benchmarks/EmbeddedServer.swift index 1565ce158..dc21fe2f6 100644 --- a/Sources/GRPCPerformanceTests/Benchmarks/EmbeddedServer.swift +++ b/Sources/GRPCPerformanceTests/Benchmarks/EmbeddedServer.swift @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import EchoModel import GRPC import Logging import NIO @@ -112,31 +111,38 @@ final class EmbeddedServerChildChannelBenchmark: Benchmark { func tearDown() throws {} - func run() throws { + func run() throws -> Int { switch self.mode { case let .unary(rpcs): - try self.run(rpcs: rpcs, requestsPerRPC: 1) + return try self.run(rpcs: rpcs, requestsPerRPC: 1) case let .clientStreaming(rpcs, requestsPerRPC): - try self.run(rpcs: rpcs, requestsPerRPC: requestsPerRPC) + return try self.run(rpcs: rpcs, requestsPerRPC: requestsPerRPC) case let .serverStreaming(rpcs, _): - try self.run(rpcs: rpcs, requestsPerRPC: 1) + return try self.run(rpcs: rpcs, requestsPerRPC: 1) case let .bidirectional(rpcs, requestsPerRPC): - try self.run(rpcs: rpcs, requestsPerRPC: requestsPerRPC) + return try self.run(rpcs: rpcs, requestsPerRPC: requestsPerRPC) } } - func run(rpcs: Int, requestsPerRPC: Int) throws { + func run(rpcs: Int, requestsPerRPC: Int) throws -> Int { + var messages = 0 for _ in 0 ..< rpcs { let channel = try self.makeChannel() try channel.writeInbound(self.headersPayload) for _ in 0 ..< (requestsPerRPC - 1) { + messages += 1 try channel.writeInbound(self.requestPayload) } + messages += 1 try channel.writeInbound(self.requestPayloadWithEndStream) while try channel.readOutbound(as: HTTP2Frame.FramePayload.self) != nil { () } + + _ = try channel.finish() } + + return messages } } diff --git a/Sources/GRPCPerformanceTests/Benchmarks/MinimalEchoProvider.swift b/Sources/GRPCPerformanceTests/Benchmarks/MinimalEchoProvider.swift index 1a6dd13ba..3d1e3a1c4 100644 --- a/Sources/GRPCPerformanceTests/Benchmarks/MinimalEchoProvider.swift +++ b/Sources/GRPCPerformanceTests/Benchmarks/MinimalEchoProvider.swift @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import EchoModel import GRPC import NIO diff --git a/Sources/GRPCPerformanceTests/Benchmarks/PercentEncoding.swift b/Sources/GRPCPerformanceTests/Benchmarks/PercentEncoding.swift index 1313ba735..61945c55c 100644 --- a/Sources/GRPCPerformanceTests/Benchmarks/PercentEncoding.swift +++ b/Sources/GRPCPerformanceTests/Benchmarks/PercentEncoding.swift @@ -38,7 +38,7 @@ class PercentEncoding: Benchmark { func tearDown() throws {} - func run() throws { + func run() throws -> Int { var totalLength: Int = 0 for _ in 0 ..< self.iterations { @@ -50,5 +50,7 @@ class PercentEncoding: Benchmark { totalLength += unmarshalled.count } + + return totalLength } } diff --git a/Sources/GRPCPerformanceTests/Benchmarks/ServerProvidingBenchmark.swift b/Sources/GRPCPerformanceTests/Benchmarks/ServerProvidingBenchmark.swift index 3115804f7..d78ba7272 100644 --- a/Sources/GRPCPerformanceTests/Benchmarks/ServerProvidingBenchmark.swift +++ b/Sources/GRPCPerformanceTests/Benchmarks/ServerProvidingBenchmark.swift @@ -41,7 +41,7 @@ class ServerProvidingBenchmark: Benchmark { try self.group.syncShutdownGracefully() } - func run() throws { - // no-op + func run() throws -> Int { + return 0 } } diff --git a/Sources/GRPCPerformanceTests/Benchmarks/UnaryThroughput.swift b/Sources/GRPCPerformanceTests/Benchmarks/UnaryThroughput.swift index 50d452f68..1a242bd30 100644 --- a/Sources/GRPCPerformanceTests/Benchmarks/UnaryThroughput.swift +++ b/Sources/GRPCPerformanceTests/Benchmarks/UnaryThroughput.swift @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import EchoModel import Foundation import GRPC import NIO @@ -44,7 +43,8 @@ class Unary: ServerProvidingBenchmark { self.client = .init(channel: channel) } - override func run() throws { + override func run() throws -> Int { + var messages = 0 let batchSize = 100 for lowerBound in stride(from: 0, to: self.requestCount, by: batchSize) { @@ -53,8 +53,12 @@ class Unary: ServerProvidingBenchmark { let requests = (lowerBound ..< upperBound).map { _ in client.get(Echo_EchoRequest.with { $0.text = self.requestText }).response } + + messages += requests.count try EventLoopFuture.andAllSucceed(requests, on: self.group.next()).wait() } + + return messages } override func tearDown() throws { @@ -73,17 +77,20 @@ class Bidi: Unary { super.init(requests: requests, text: text) } - override func run() throws { + override func run() throws -> Int { + var messages = 0 let update = self.client.update { _ in } for _ in stride(from: 0, to: self.requestCount, by: self.batchSize) { let batch = (0 ..< self.batchSize).map { _ in Echo_EchoRequest.with { $0.text = self.requestText } } + messages += batch.count update.sendMessages(batch, promise: nil) } update.sendEnd(promise: nil) _ = try update.status.wait() + return messages } } diff --git a/Sources/GRPCPerformanceTests/Benchmarks/echo.grpc.swift b/Sources/GRPCPerformanceTests/Benchmarks/echo.grpc.swift new file mode 120000 index 000000000..4461d497d --- /dev/null +++ b/Sources/GRPCPerformanceTests/Benchmarks/echo.grpc.swift @@ -0,0 +1 @@ +../../Examples/Echo/Model/echo.grpc.swift \ No newline at end of file diff --git a/Sources/GRPCPerformanceTests/Benchmarks/echo.pb.swift b/Sources/GRPCPerformanceTests/Benchmarks/echo.pb.swift new file mode 120000 index 000000000..b4fd86913 --- /dev/null +++ b/Sources/GRPCPerformanceTests/Benchmarks/echo.pb.swift @@ -0,0 +1 @@ +../../Examples/Echo/Model/echo.pb.swift \ No newline at end of file diff --git a/Sources/GRPCPerformanceTests/main.swift b/Sources/GRPCPerformanceTests/main.swift index 3809b7dc5..d38540338 100644 --- a/Sources/GRPCPerformanceTests/main.swift +++ b/Sources/GRPCPerformanceTests/main.swift @@ -14,7 +14,6 @@ * limitations under the License. */ import ArgumentParser -import EchoModel import Foundation import GRPC import Logging diff --git a/scripts/alloc-limits.sh b/scripts/alloc-limits.sh new file mode 100755 index 000000000..5c66b3d39 --- /dev/null +++ b/scripts/alloc-limits.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2021, gRPC Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script parses output from the SwiftNIO allocation counter framework to +# generate a list of per-test limits for total_allocations. +# +# Input is like: +# ... +# test_embedded_server_unary_1k_rpcs_1_small_request.total_allocated_bytes: 5992858 +# test_embedded_server_unary_1k_rpcs_1_small_request.total_allocations: 63000 +# test_embedded_server_unary_1k_rpcs_1_small_request.remaining_allocations: 0 +# DEBUG: [["total_allocated_bytes": 5992858, "total_allocations": ... +# +# Output: +# MAX_ALLOCS_ALLOWED_embedded_server_unary_1k_rpcs_1_small_request=64000 + +grep 'test_.*\.total_allocations: ' \ + | sed 's/^test_/MAX_ALLOCS_ALLOWED_/' \ + | sed 's/.total_allocations://' \ + | awk '{ print $1 "=" ((int($2 / 1000) + 1) * 1000) }' \ + | sort