Add allocation counting tests

.travis-script.sh

@@ -56,6 +56,15 @@ make_test() {
   echo -en 'travis_fold:end:make.test\\r'
 }
 
+run_allocation_tests() {
+  echo -en 'travis_fold:start:allocation_tests\\r'
+
+  info "Running allocation counter tests"
+  ./Performance/allocations/test-allocation-counts.sh
+
+  echo -en 'travis_fold:end:allocation_tests\\r'
+}
+
 make_test_plugin() {
   echo -en 'travis_fold:start:make.test_plugin\\r'
   info "Validating protoc plugins on the Echo service"
@@ -152,10 +161,11 @@ run_interop_reconnect_test() {
 }
 
 just_sanity=false
+allocation_tests=false
 just_interop_tests=false
 tsan=false
 
-while getopts "sit" optname; do
+while getopts "sita" optname; do
   case $optname in
     s)
       just_sanity=true
@@ -166,6 +176,9 @@ while getopts "sit" optname; do
     t)
       tsan=true
       ;;
+    a)
+      allocation_tests=true
+      ;;
     \?)
       echo "Uknown option $optname"
       exit 2
@@ -183,6 +196,9 @@ elif $just_interop_tests; then
 else
   make_all
   make_test $tsan
+  if $allocation_tests; then
+    run_allocation_tests
+  fi
   make_test_plugin
   make_project
 fi

.travis.yml

@@ -26,20 +26,28 @@ jobs:
       dist: bionic
       install: ./.travis-install.sh -f  # install swiftformat
       script: ./.travis-script.sh -s  # just sanity
-      env: SWIFT_VERSION=5.3.1
+      env: SWIFT_VERSION=5.3.3
     # Tests for each PR.
     - &tests
       stage: "Test"
       name: "Unit Tests: Ubuntu 18.04 (Swift 5.3)"
       os: linux
       dist: bionic
       install: ./.travis-install.sh -p  # install protoc
-      script: ./.travis-script.sh -t  # with tsan
-      env: SWIFT_VERSION=5.3.1
+      script: ./.travis-script.sh -t -a # tests with tsan, run allocation tests
+      env:
+        - SWIFT_VERSION=5.3.3
+        - MAX_ALLOCS_ALLOWED_embedded_server_bidi_1k_rpcs_10_small_requests=113000
+        - MAX_ALLOCS_ALLOWED_embedded_server_bidi_1k_rpcs_1_small_request=68000
+        - MAX_ALLOCS_ALLOWED_embedded_server_unary_1k_rpcs_1_small_request=64000
     - <<: *tests
       name: "Unit Tests: Ubuntu 18.04 (Swift 5.2)"
-      script: ./.travis-script.sh
-      env: SWIFT_VERSION=5.2.5
+      script: ./.travis-script.sh -a # run allocation tests
+      env:
+        - SWIFT_VERSION=5.2.5
+        - MAX_ALLOCS_ALLOWED_embedded_server_bidi_1k_rpcs_10_small_requests=113000
+        - MAX_ALLOCS_ALLOWED_embedded_server_bidi_1k_rpcs_1_small_request=68000
+        - MAX_ALLOCS_ALLOWED_embedded_server_unary_1k_rpcs_1_small_request=64000
     - <<: *tests
       name: "Unit Tests: Xcode 12.2"
       os: osx
@@ -57,7 +65,7 @@ jobs:
       dist: bionic
       install: ./.travis-install.sh -p -i # install protoc and interop server
       script: ./.travis-script.sh -i  # interop tests
-      env: SWIFT_VERSION=5.3.1
+      env: SWIFT_VERSION=5.3.3
     - <<: *interop_tests
       name: "Interoperability Tests: Ubuntu 18.04 (Swift 5.2)"
       env: SWIFT_VERSION=5.2.5

NOTICES.txt

@@ -0,0 +1,27 @@
+                          The gRPC Swift Project
+                          ======================
+
+Copyright 2021 The gRPC Swift Project
+
+The gRPC Swift project licenses this file to you under the Apache License,
+version 2.0 (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at:
+
+  https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+
+-------------------------------------------------------------------------------
+
+This product uses scripts derived from SwiftNIO's integration testing
+framework: 'test_01_allocation_counts.sh', 'run-nio-alloc-counter-tests.sh' and
+'test_functions.sh'.
+
+  * LICENSE (Apache License 2.0):
+    * https://github.com/apple/swift-nio/blob/main/LICENSE.txt
+  * HOMEPAGE:
+    * https://github.com/apple/swift-nio

Package.swift

@@ -140,7 +140,6 @@ let package = Package(
       name: "GRPCPerformanceTests",
       dependencies: [
         .target(name: "GRPC"),
-        .target(name: "EchoModel"),
         .product(name: "NIO", package: "swift-nio"),
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
       ]

Performance/allocations/test-allocation-counts.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Copyright 2021, gRPC Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script was adapted from SwiftNIO's test_01_allocation_counts.sh. The
+# license for the original work is reproduced below. See NOTICES.txt for more.
+
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+set -eu
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+tmp="/tmp"
+
+source "$here/test-utils.sh"
+
+all_tests=()
+for file in "$here/tests/"test_*.swift; do
+  # Extract the "TESTNAME" from "test_TESTNAME.swift"
+  test_name=$(basename "$file")
+  test_name=${test_name#test_*}
+  test_name=${test_name%*.swift}
+  all_tests+=( "$test_name" )
+done
+
+# Run all the tests.
+"$here/tests/run-allocation-counter-tests.sh" -t "$tmp" > "$tmp/output"
+
+# Dump some output from each, check for allocations.
+for test in "${all_tests[@]}"; do
+  cat "$tmp/output"  # helps debugging
+
+  while read -r test_case; do
+    test_case=${test_case#test_*}
+    total_allocations=$(grep "^test_$test_case.total_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
+    not_freed_allocations=$(grep "^test_$test_case.remaining_allocations:" "$tmp/output" | cut -d: -f2 | sed 's/ //g')
+    max_allowed_env_name="MAX_ALLOCS_ALLOWED_$test_case"
+
+    info "$test_case: allocations not freed: $not_freed_allocations"
+    info "$test_case: total number of mallocs: $total_allocations"
+
+    assert_less_than "$not_freed_allocations" 5     # allow some slack
+    assert_greater_than "$not_freed_allocations" -5 # allow some slack
+    if [[ -z "${!max_allowed_env_name+x}" ]]; then
+      if [[ -z "${!max_allowed_env_name+x}" ]]; then
+        warn "no reference number of allocations set (set to \$$max_allowed_env_name)"
+        warn "to set current number:"
+        warn "    export $max_allowed_env_name=$total_allocations"
+      fi
+    else
+      max_allowed=${!max_allowed_env_name}
+      assert_less_than_or_equal "$total_allocations" "$max_allowed"
+      assert_greater_than "$total_allocations" "$(( max_allowed - 1000))"
+    fi
+  done < <(grep "^test_$test[^\W]*.total_allocations:" "$tmp/output" | cut -d: -f1 | cut -d. -f1 | sort | uniq)
+done

Performance/allocations/test-utils.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright 2021, gRPC Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script contains part of SwiftNIO's test_functions.sh script. The license
+# for the original work is reproduced below. See NOTICES.txt for more.
+
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+function fail() {
+  echo >&2 "FAILURE: $*"
+  false
+}
+
+function assert_less_than() {
+  if [[ ! "$1" -lt "$2" ]]; then
+    fail "assertion '$1' < '$2' failed"
+  fi
+}
+
+function assert_less_than_or_equal() {
+  if [[ ! "$1" -le "$2" ]]; then
+    fail "assertion '$1' <= '$2' failed"
+  fi
+}
+
+function assert_greater_than() {
+  if [[ ! "$1" -gt "$2" ]]; then
+    fail "assertion '$1' > '$2' failed"
+  fi
+}
+
+g_has_previously_infoed=false
+
+function info() {
+  if ! $g_has_previously_infoed; then
+    echo || true # echo an extra newline so it looks better
+    g_has_previously_infoed=true
+  fi
+  echo "info: $*" || true
+}
+
+function warn() {
+  echo "warning: $*"
+}

Performance/allocations/tests/run-allocation-counter-tests.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Copyright 2021, gRPC Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script was adapted from SwiftNIO's 'run-nio-alloc-counter-tests.sh'
+# script. The license for the original work is reproduced below. See NOTICES.txt
+# for more.
+
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+set -eu
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+tmp_dir="/tmp"
+
+while getopts "t:" opt; do
+  case "$opt" in
+    t)
+      tmp_dir="$OPTARG"
+      ;;
+    *)
+      exit 1
+      ;;
+  esac
+done
+
+nio_checkout=$(mktemp -d "$tmp_dir/.swift-nio_XXXXXX")
+(
+cd "$nio_checkout"
+git clone --depth 1 https://github.com/apple/swift-nio
+)
+
+shift $((OPTIND-1))
+
+tests_to_run=("$here"/test_*.swift)
+
+if [[ $# -gt 0 ]]; then
+  tests_to_run=("$@")
+fi
+
+# We symlink in a bunch of components from the GRPCPerformanceTests target to
+# avoid duplicating a bunch of code.
+"$nio_checkout/swift-nio/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh" \
+  -p "$here/../../.." \
+  -m GRPC \
+  -t "$tmp_dir" \
+  -s "$here/shared/Benchmark.swift" \
+  -s "$here/shared/echo.pb.swift" \
+  -s "$here/shared/echo.grpc.swift" \
+  -s "$here/shared/MinimalEchoProvider.swift" \
+  -s "$here/shared/EmbeddedServer.swift" \
+  "${tests_to_run[@]}"

Performance/allocations/tests/shared/Benchmark.swift

@@ -0,0 +1 @@
+../../../../Sources/GRPCPerformanceTests/Benchmark.swift

Performance/allocations/tests/shared/EmbeddedServer.swift

@@ -0,0 +1 @@
+../../../../Sources/GRPCPerformanceTests/Benchmarks/EmbeddedServer.swift

Performance/allocations/tests/shared/MinimalEchoProvider.swift

@@ -0,0 +1 @@
+../../../../Sources/GRPCPerformanceTests/Benchmarks/MinimalEchoProvider.swift

Performance/allocations/tests/shared/echo.grpc.swift

@@ -0,0 +1 @@
+../../../../Sources/GRPCPerformanceTests/Benchmarks/echo.grpc.swift

Performance/allocations/tests/shared/echo.pb.swift

@@ -0,0 +1 @@
+../../../../Sources/GRPCPerformanceTests/Benchmarks/echo.pb.swift

Performance/allocations/tests/test_embedded_server_bidi_1k_rpcs_10_small_requests.swift

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2021, gRPC Authors All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+func run(identifier: String) {
+  measure(identifier: identifier) {
+    let benchmark = EmbeddedServerChildChannelBenchmark(
+      mode: .bidirectional(rpcs: 1000, requestsPerRPC: 10),
+      text: ""
+    )
+    return try! benchmark.runOnce()
+  }
+}