Run benchmarks on CI. (open-telemetry#34)

.circleci/config.yml

@@ -48,6 +48,19 @@ jobs:
       - run: ./ci/install_bazelisk.sh
       - run: ./ci/do_ci.sh bazel.asan
 
+  benchmark:
+    resource_class: xlarge
+    docker:
+      - image: ubuntu:18.04
+    steps:
+      - checkout
+      - run: ./ci/setup_ci_environment.sh
+      - run: ./ci/install_bazelisk.sh
+      - run: env BENCHMARK_DIR=/benchmark ./ci/do_ci.sh benchmark
+      - store_artifacts:
+          path: /benchmark
+          destination: benchmark
+
   format:
     resource_class: xlarge
     docker:
@@ -83,5 +96,6 @@ workflows:
       - bazel_test
       - bazel_asan
       - format
+      - benchmark
       - osx_test
       - windows

api/test/trace/BUILD

@@ -1,3 +1,5 @@
+load("//bazel:otel_cc_benchmark.bzl", "otel_cc_benchmark")
+
 cc_test(
     name = "noop_test",
     srcs = [
@@ -9,15 +11,10 @@ cc_test(
     ],
 )
 
-cc_binary(
+otel_cc_benchmark(
     name = "span_id_benchmark",
-    srcs = [
-        "span_id_benchmark.cc",
-    ],
-    deps = [
-        "//api",
-        "@com_github_google_benchmark//:benchmark",
-    ],
+    srcs = ["span_id_benchmark.cc"],
+    deps = ["//api"],
 )
 
 cc_test(

bazel/BUILD

@@ -0,0 +1 @@
+package(default_visibility = ["//:__subpackages__"])

bazel/otel_cc_benchmark.bzl

@@ -0,0 +1,48 @@
+def otel_cc_benchmark(name, srcs, deps):
+    """
+    Creates targets for the benchmark and related targets.
+
+    Example:
+
+      otel_cc_benchmark(
+          name = "foo_benchmark",
+          srcs = ["foo_benchmark.cc"],
+          deps = ["//bar"],
+      )
+
+    Creates:
+
+      :foo_benchmark           (the benchmark binary)
+      :foo_benchmark_result    (results from running the benchmark)
+      :foo_benchmark_smoketest (a fast test that runs a single iteration)
+    """
+
+    # This is the benchmark as a binary, it can be run manually, and is used
+    # to generate the _result below.
+    native.cc_binary(
+        name = name,
+        srcs = srcs,
+        deps = deps + ["@com_github_google_benchmark//:benchmark"],
+        tags = ["manual"],
+    )
+
+    # The result of running the benchmark, captured into a text file.
+    native.genrule(
+        name = name + "_result",
+        outs = [name + "_result.txt"],
+        tools = [":" + name],
+        tags = ["benchmark_result", "manual"],
+        testonly = True,
+        cmd = "$(location :" + name + (") --benchmark_color=false " +
+                                       "--benchmark_min_time=.1 &> $@"),
+    )
+
+    # This is run as part of "bazel test ..." to smoke-test benchmarks. It's
+    # meant to complete quickly rather than get accurate results.
+    native.cc_test(
+        name = name + "_smoketest",
+        srcs = srcs,
+        deps = deps + ["@com_github_google_benchmark//:benchmark"],
+        args = ["--benchmark_min_time=0"],
+        tags = ["benchmark"],
+    )

ci/README.md

@@ -1,9 +1,11 @@
-##  Building and running tests as a developer
+## Building and running tests as a developer
+
 CI tests can be run on docker by invoking the script `./ci/run_docker.sh ./ci/do_ci.sh <TARGET>` where the targets are:
 
 * `cmake.test`: build cmake targets and run tests.
 * `bazel.test`: build bazel targets and run tests.
 * `bazel.asan`: build bazel targets and run tests with AddressSanitizer.
+* `benchmark`: run all benchmarks.
 * `format`: use `tools/format.sh` to enforce text formatting.
 
 Additionally, `./ci/run_docker.sh` can be invoked with no arguments to get a docker shell where tests

ci/do_ci.sh

@@ -24,6 +24,19 @@ elif [[ "$1" == "bazel.test" ]]; then
 elif [[ "$1" == "bazel.asan" ]]; then
   bazel test --config=asan $BAZEL_TEST_OPTIONS //...
   exit 0
+elif [[ "$1" == "benchmark" ]]; then
+  [ -z "${BENCHMARK_DIR}" ] && export BENCHMARK_DIR=$HOME/benchmark
+  bazel build $BAZEL_OPTIONS -c opt -- \
+    $(bazel query 'attr("tags", "benchmark_result", ...)')
+  echo ""
+  echo "Benchmark results in $BENCHMARK_DIR:"
+  (
+    cd bazel-bin
+    find . -name \*_result.txt -exec bash -c \
+      'echo "$@" && mkdir -p "$BENCHMARK_DIR/$(dirname "$@")" && \
+       cp "$@" "$BENCHMARK_DIR/$@" && chmod +w "$BENCHMARK_DIR/$@"' _ {} \;
+  )
+  exit 0
 elif [[ "$1" == "format" ]]; then
   tools/format.sh
   CHANGED="$(git ls-files --modified)"