ci: enable clang based RBE in Azure pipelines

.azure-pipelines/linux.yml

@@ -10,20 +10,27 @@ jobs:
     vmImage: 'Ubuntu 16.04'
   container: envoy-build
   steps:
-  # bazel.dev isn't for CI purpose but we use it here to experiment with Azure Pipeline
-  # as it builds faster, before Azure can provision larger VM or we can use RBE.
-  - script: ci/do_ci.sh bazel.dev
+  - bash: |
+      echo "disk space at beginning of build:"
+      df -h
+
+  - script: ci/do_ci.sh bazel.release
     env:
       ENVOY_SRCDIR: $(Build.SourcesDirectory)
       BUILD_DIR: $(Build.StagingDirectory)
-      BAZEL_EXTRA_TEST_OPTIONS: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only"
-      BAZEL_REMOTE_CACHE: remotebuildexecution.googleapis.com
+      BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-clang --config=remote-ci --jobs=100 --curses=no"
+      # //test/integration:echo_integration_test is marked exclusive so it will be run locally, no IPv6 locally in container.
+      BAZEL_EXTRA_TEST_OPTIONS: "--test_filter=-IpVersions/EchoIntegrationTest.*/IPv6"
+      BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
       BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
       GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
 
-  - task: PublishTestResults@2
-    inputs:
-      testResultsFiles: '**/testlogs/**/test.xml'
-      testRunTitle: 'bazel.dev'
-      searchFolder: $(Build.StagingDirectory)
+  - bash: |
+      echo "disk space at end of build:"
+      df -h
     condition: always()
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathtoPublish: "$(Build.StagingDirectory)/envoy"
+      artifactName: 'envoy'

.azure-pipelines/macos.yml

@@ -15,7 +15,7 @@ jobs:
   - script: ./ci/mac_ci_steps.sh
     displayName: 'Run Mac CI'
     env:
-      BAZEL_REMOTE_CACHE: remotebuildexecution.googleapis.com
+      BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
       BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
       GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
 

.bazelrc

@@ -70,3 +70,47 @@ build:sizeopt -c opt --copt -Os
 
 # Test options
 build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH
+
+# Remote execution: https://docs.bazel.build/versions/master/remote-execution.html
+build:rbe-toolchain --host_javabase=@rbe_ubuntu_clang//java:jdk
+build:rbe-toolchain --javabase=@rbe_ubuntu_clang//java:jdk
+build:rbe-toolchain --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:rbe-toolchain --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:rbe-toolchain --host_platform=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform
+build:rbe-toolchain --platforms=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform
+build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+build:rbe-toolchain --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
+build:rbe-toolchain --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
+build:rbe-toolchain --linkopt=-fuse-ld=lld
+build:rbe-toolchain --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+
+build:remote --spawn_strategy=remote,sandboxed,local
+build:remote --strategy=Javac=remote,sandboxed,local
+build:remote --strategy=Closure=remote,sandboxed,local
+build:remote --strategy=Genrule=remote,sandboxed,local
+build:remote --remote_timeout=3600
+build:remote --auth_enabled=true
+build:remote --experimental_inmemory_jdeps_files
+build:remote --experimental_inmemory_dotd_files
+build:remote --experimental_remote_download_outputs=toplevel
+test:remote --experimental_remote_download_outputs=minimal
+
+build:remote-clang --config=remote
+build:remote-clang --config=rbe-toolchain
+
+# Docker sandbox
+build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build:cfc514546bc0284536893cca5fa43d7128edcd35
+build:docker-sandbox --spawn_strategy=docker
+build:docker-sandbox --strategy=Javac=docker
+build:docker-sandbox --strategy=Closure=docker
+build:docker-sandbox --strategy=Genrule=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+
+build:docker-clang --config=docker-sandbox
+build:docker-clang --config=rbe-toolchain
+
+# CI configurations
+build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com
+build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com

.circleci/config.yml

@@ -4,6 +4,7 @@ executors:
   ubuntu-build:
     description: "A regular build executor based on ubuntu image"
     docker:
+      # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here.
       - image: envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d
     resource_class: xlarge
     working_directory: /source

WORKSPACE

@@ -17,3 +17,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
 go_rules_dependencies()
 
 go_register_toolchains(go_version = GO_VERSION)
+
+load("@envoy//bazel/toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config")
+
+rbe_toolchains_config()

bazel/BUILD

@@ -5,6 +5,7 @@ package(default_visibility = ["//visibility:public"])
 exports_files([
     "gen_sh_test_runner.sh",
     "sh_test_wrapper.sh",
+    "cc_wrapper.py",
 ])
 
 genrule(

bazel/repositories.bzl

@@ -156,6 +156,7 @@ def envoy_dependencies(skip_targets = []):
     _com_lightstep_tracer_cpp()
     _io_opentracing_cpp()
     _net_zlib()
+    _repository_impl("bazel_toolchains")
 
     _python_deps()
     _cc_deps()

bazel/repository_locations.bzl

@@ -3,6 +3,14 @@ REPOSITORY_LOCATIONS = dict(
         sha256 = "3c681998538231a2d24d0c07ed5a7658cb72bfb5fd4bf9911157c0e9ac6a2687",
         urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.17.0/bazel-gazelle-0.17.0.tar.gz"],
     ),
+    bazel_toolchains = dict(
+        sha256 = "68e7678473090542e679ce7e6aa8a3ba5669577dede2b404f9865d556bd99f10",
+        strip_prefix = "bazel-toolchains-0.28.0",
+        urls = [
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/0.28.0.tar.gz",
+            "https://github.com/bazelbuild/bazel-toolchains/archive/0.28.0.tar.gz",
+        ],
+    ),
     boringssl = dict(
         # Use commits from branch "chromium-stable-with-bazel"
         sha256 = "448773376d063b1e9a19e4fd41002d1a31a968a13be20b3b66ecd4aab9cf14a8",

bazel/toolchains/BUILD

@@ -0,0 +1,17 @@
+licenses(["notice"])  # Apache 2
+
+platform(
+    name = "rbe_ubuntu_clang_platform",
+    parents = ["@rbe_ubuntu_clang//config:platform"],
+    remote_execution_properties = """
+        {PARENT_REMOTE_EXECUTION_PROPERTIES}
+        properties: {
+          name: "dockerAddCapabilities"
+          value: "SYS_PTRACE,NET_RAW,NET_ADMIN"
+        }
+        properties: {
+          name: "dockerNetwork"
+          value: "standard"
+        }
+        """,
+)

bazel/toolchains/README.md

@@ -0,0 +1,13 @@
+# Bazel Toolchains
+
+This directory contains toolchains config generated for Bazel [RBE](https://docs.bazel.build/versions/master/remote-execution.html) and
+[Docker sandbox](https://docs.bazel.build/versions/master/remote-execution-sandbox.html).
+
+To regenerate toolchain configs, update the docker image information in `rbe_toolchains_config.bzl` and run following command in an
+environment with the latest Bazel and Docker installed:
+
+```
+bazel/toolchains/regenerate.sh
+```
+
+This will generate configs in `bazel/toolchains/configs`, check in those files so they can be used in CI.

bazel/toolchains/configs/.gitignore

@@ -0,0 +1,2 @@
+# RBE autoconfig generator will generate a .bazelrc but we don't need it.
+.latest.bazelrc

bazel/toolchains/configs/clang/bazel_0.28.0/cc/BUILD

@@ -0,0 +1,148 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes.
+
+package(default_visibility = ["//visibility:public"])
+
+load(":cc_toolchain_config.bzl", "cc_toolchain_config")
+load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config")
+
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "malloc",
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+)
+
+filegroup(
+    name = "cc_wrapper",
+    srcs = ["cc_wrapper.sh"],
+)
+
+filegroup(
+    name = "compiler_deps",
+    srcs = glob(["extra_tools/**"], allow_empty = True) + [":empty"],
+)
+
+# This is the entry point for --crosstool_top.  Toolchains are found
+# by lopping off the name of --crosstool_top and searching for
+# the "${CPU}" entry in the toolchains attribute.
+cc_toolchain_suite(
+    name = "toolchain",
+    toolchains = {
+        "k8|clang": ":cc-compiler-k8",
+        "k8": ":cc-compiler-k8",
+        "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a",
+        "armeabi-v7a": ":cc-compiler-armeabi-v7a",
+    },
+)
+
+cc_toolchain(
+    name = "cc-compiler-k8",
+    toolchain_identifier = "local",
+    toolchain_config = ":local",
+    all_files = ":compiler_deps",
+    ar_files = ":compiler_deps",
+    as_files = ":compiler_deps",
+    compiler_files = ":compiler_deps",
+    dwp_files = ":empty",
+    linker_files = ":compiler_deps",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    supports_param_files = 1,
+)
+
+cc_toolchain_config(
+    name = "local",
+    cpu = "k8",
+    compiler = "clang",
+    toolchain_identifier = "local",
+    host_system_name = "local",
+    target_system_name = "local",
+    target_libc = "local",
+    abi_version = "local",
+    abi_libc_version = "local",
+    cxx_builtin_include_directories = ["/usr/local/include",
+    "/usr/lib/llvm-8/lib/clang/8.0.1/include",
+    "/usr/include/x86_64-linux-gnu",
+    "/usr/include",
+    "/usr/include/c++/7.4.0",
+    "/usr/include/x86_64-linux-gnu/c++/7.4.0",
+    "/usr/include/c++/7.4.0/backward",
+    "/usr/include/clang/8.0.1/include"],
+    tool_paths = {"ar": "/usr/bin/ar",
+        "ld": "/usr/bin/ld",
+        "cpp": "/usr/bin/cpp",
+        "gcc": "/usr/lib/llvm-8/bin/clang",
+        "dwp": "/usr/bin/dwp",
+        "gcov": "/usr/lib/llvm-8/bin/llvm-profdata",
+        "nm": "/usr/bin/nm",
+        "objcopy": "/usr/bin/objcopy",
+        "objdump": "/usr/bin/objdump",
+        "strip": "/usr/bin/strip"},
+    compile_flags = ["-U_FORTIFY_SOURCE",
+    "-fstack-protector",
+    "-Wall",
+    "-Wthread-safety",
+    "-Wself-assign",
+    "-fcolor-diagnostics",
+    "-fno-omit-frame-pointer"],
+    opt_compile_flags = ["-g0",
+    "-O2",
+    "-D_FORTIFY_SOURCE=1",
+    "-DNDEBUG",
+    "-ffunction-sections",
+    "-fdata-sections"],
+    dbg_compile_flags = ["-g"],
+    cxx_flags = ["-std=c++0x"],
+    link_flags = ["-fuse-ld=gold",
+    "-Wl,-no-as-needed",
+    "-Wl,-z,relro,-z,now",
+    "-B/usr/lib/llvm-8/bin",
+    "-lm",
+    "-static-libgcc"],
+    link_libs = ["-l:libstdc++.a"],
+    opt_link_flags = ["-Wl,--gc-sections"],
+    unfiltered_compile_flags = ["-no-canonical-prefixes",
+    "-Wno-builtin-macro-redefined",
+    "-D__DATE__=\"redacted\"",
+    "-D__TIMESTAMP__=\"redacted\"",
+    "-D__TIME__=\"redacted\""],
+    coverage_compile_flags = ["-fprofile-instr-generate",  "-fcoverage-mapping"],
+    coverage_link_flags = ["-fprofile-instr-generate"],
+    supports_start_end_lib = True,
+)
+
+# Android tooling requires a default toolchain for the armeabi-v7a cpu.
+cc_toolchain(
+    name = "cc-compiler-armeabi-v7a",
+    toolchain_identifier = "stub_armeabi-v7a",
+    toolchain_config = ":stub_armeabi-v7a",
+    all_files = ":empty",
+    ar_files = ":empty",
+    as_files = ":empty",
+    compiler_files = ":empty",
+    dwp_files = ":empty",
+    linker_files = ":empty",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    supports_param_files = 1,
+)
+
+armeabi_cc_toolchain_config(name = "stub_armeabi-v7a")