Merge pull request #100 from PiotrSikora/sync-1.3

Sync with envoy-wasm/master (2019-08-27).

.bazelrc

@@ -28,6 +28,7 @@ build --action_env=PATH
 # Basic ASAN/UBSAN that works for gcc
 build:asan --action_env=BAZEL_LINKLIBS=
 build:asan --action_env=BAZEL_LINKOPTS=-lstdc++:-lm
+build:asan --action_env=ENVOY_ASAN=1
 build:asan --define ENVOY_CONFIG_ASAN=1
 build:asan --copt -fsanitize=address,undefined
 build:asan --linkopt -fsanitize=address,undefined
@@ -59,6 +60,7 @@ build:macos-asan --copt -DGRPC_BAZEL_BUILD
 build:macos-asan --dynamic_mode=off
 
 # Clang TSAN
+build:clang-tsan --action_env=ENVOY_TSAN=1
 build:clang-tsan --define ENVOY_CONFIG_TSAN=1
 build:clang-tsan --copt -fsanitize=thread
 build:clang-tsan --linkopt -fsanitize=thread
@@ -70,6 +72,7 @@ build:clang-tsan --copt -DEVENT__DISABLE_DEBUG_MODE
 
 # Clang MSAN - broken today since we need to rebuild lib[std]c++ and external deps with MSAN
 # support (see https://github.com/envoyproxy/envoy/issues/443).
+build:clang-msan --action_env=ENVOY_MSAN=1
 build:clang-msan --define ENVOY_CONFIG_MSAN=1
 build:clang-msan --copt -fsanitize=memory
 build:clang-msan --linkopt -fsanitize=memory
@@ -80,6 +83,7 @@ build:clang-msan --copt -fsanitize-memory-track-origins=2
 # TODO(cmluciano) fix and re-enable _LIBCPP_VERSION testing for TCMALLOC in Envoy::Stats::TestUtil::hasDeterministicMallocStats
 # and update stats_integration_test with appropriate m_per_cluster value
 build:libc++ --action_env=CXXFLAGS=-stdlib=libc++
+build:libc++ --action_env=LDFLAGS=-stdlib=libc++
 build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++
 build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a:-lm
 build:libc++ --host_linkopt=-fuse-ld=lld
@@ -105,6 +109,8 @@ build:rbe-toolchain-clang-libc++ --config=rbe-toolchain
 build:rbe-toolchain-clang-libc++ --crosstool_top=@rbe_ubuntu_clang_libcxx//cc:toolchain
 build:rbe-toolchain-clang-libc++ --extra_toolchains=@rbe_ubuntu_clang_libcxx//config:cc-toolchain
 build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin
+build:rbe-toolchain-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++
+build:rbe-toolchain-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++
 
 build:rbe-toolchain-gcc --config=rbe-toolchain
 build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain

.circleci/config.yml

@@ -73,10 +73,12 @@ jobs:
 
    macos:
      macos:
-       xcode: "10.2.1"
+       xcode: "11.0.0"
      environment:
        BAZEL_BUILD_EXTRA_OPTIONS: "--define wasm=v8" # v8 only, WAVM segfaults in tests.
        BAZEL_TEST_TARGETS: "//test/extensions/access_loggers/wasm/... //test/extensions/filters/http/wasm/... //test/extensions/wasm/..."
+       CC: clang
+       CXX: clang++
      steps:
        - run: sudo sntp -sS time.apple.com
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken

api/wasm/cpp/proxy_wasm_api.h

@@ -49,12 +49,22 @@ inline void logAbort(StringView logMessag) {
   abort();
 }
 
+#define LOG(_level, ...)                                                        \
+    log##_level(std::string("[") + __FILE__ + ":" + std::to_string(__LINE__) + "]::" + __FUNCTION__ + "() " + __VA_ARGS__)
+#define LOG_TRACE(...) LOG(Trace, __VA_ARGS__)
+#define LOG_DEBUG(...) LOG(Debug, __VA_ARGS__)
+#define LOG_INFO(...) LOG(Info, __VA_ARGS__)
+#define LOG_WARN(...) LOG(Warn, __VA_ARGS__)
+#define LOG_ERROR(...) LOG(Error, __VA_ARGS__)
+#define LOG_CRITICAL(...) LOG(Critical, __VA_ARGS__)
+
 // Buffers coming into the WASM filter.
 class WasmData {
 public:
   WasmData(const char* data, size_t size) : data_(data), size_(size) {}
   ~WasmData() { ::free(const_cast<char*>(data_)); }
   const char* data() { return data_; }
+  size_t size() { return size_; }
   StringView view() { return {data_, size_}; }
   std::string toString() { return std::string(view()); }
   std::vector<std::pair<StringView, StringView>> pairs();
@@ -528,6 +538,32 @@ inline WasmResult getPluginDirection(PluginDirection *direction_ptr) {
   PROXY_EXPRESSION_GET("plugin.direction", reinterpret_cast<uint32_t*>(direction_ptr));
 }
 
+// Generic selector
+inline Optional<WasmDataPtr> getSelectorExpression(std::initializer_list<StringView> parts) {
+  size_t size = 0;
+  for (auto part: parts) {
+    size += part.size() + 1; // null terminated string value
+  }
+
+  char* buffer = static_cast<char*>(::malloc(size));
+  char* b = buffer;
+
+  for (auto part : parts) {
+    memcpy(b, part.data(), part.size());
+    b += part.size();
+    *b++ = 0;
+  }
+
+  const char* value_ptr = nullptr;
+  size_t value_size = 0;
+  auto result = proxy_getSelectorExpression(buffer, size, &value_ptr, &value_size);
+  ::free(buffer);
+  if (result != WasmResult::Ok) {
+    return {};
+  }
+  return std::make_unique<WasmData>(value_ptr, value_size);
+}
+
 // Metadata
 inline WasmResult getMetadata(MetadataType type, StringView key, WasmDataPtr *wasm_data) {
   const char* value_ptr = nullptr;

api/wasm/cpp/proxy_wasm_externs.h

@@ -72,6 +72,10 @@ extern "C" WasmResult proxy_getMetadataStruct(MetadataType type, const char* nam
 extern "C" WasmResult proxy_setMetadataStruct(MetadataType type, const char* name_ptr, size_t name_size,
                                             const char* value_ptr, size_t value_size);
 
+// Generic selector
+extern "C" WasmResult proxy_getSelectorExpression(const char* path_ptr, size_t path_size,
+                                                  const char** value_ptr_ptr, size_t* value_size_ptr);
+
 // Continue/Reply/Route
 extern "C" WasmResult proxy_continueRequest();
 extern "C" WasmResult proxy_continueResponse();

api/wasm/cpp/proxy_wasm_intrinsics.h

@@ -12,6 +12,8 @@
 #include <cstdint>
 #include <string_view>
 using StringView = std::string_view;
+#include <optional>
+template <typename T> using Optional = std::optional<T>;
 
 #include "proxy_wasm_enums.h"
 #include "proxy_wasm_result.h"

bazel/BUILD

@@ -214,16 +214,6 @@ alias(
     }),
 )
 
-# Alias pointing to a platform-specific version of wee8.
-alias(
-    name = "wee8",
-    actual = select({
-        ":darwin": "@wee8_macos//:wee8",
-        ":darwin_x86_64": "@wee8_macos//:wee8",
-        "//conditions:default": "@wee8_linux//:wee8",
-    }),
-)
-
 config_setting(
     name = "linux_x86_64",
     values = {"cpu": "k8"},

bazel/external/wee8.genrule_cmd

@@ -15,8 +15,59 @@ pushd $$ROOT/wee8
 # Clean after previous build.
 rm -rf out/wee8
 
+# Export compiler configuration.
+if [[ ( `uname` == "Darwin" && $${CXX-} == "" ) || $${CXX-} == *"clang"* ]]; then
+  export IS_CLANG=true
+  export CC=$${CC:-clang}
+  export CXX=$${CXX:-clang++}
+else
+  export IS_CLANG=false
+  export CC=$${CC:-gcc}
+  export CXX=$${CXX:-g++}
+fi
+
+export AR=$${AR:-ar}
+export NM=$${NM:-nm}
+
+# Hook sanitizers.
+if [[ $${ENVOY_ASAN-} == "1" ]]; then
+  WEE8_BUILD_ARGS+=" is_asan=true"
+  WEE8_BUILD_ARGS+=" is_lsan=true"
+fi
+if [[ $${ENVOY_MSAN-} == "1" ]]; then
+  WEE8_BUILD_ARGS+=" is_msan=true"
+fi
+if [[ $${ENVOY_TSAN-} == "1" ]]; then
+  WEE8_BUILD_ARGS+=" is_tsan=true"
+fi
+
+# Release build.
+WEE8_BUILD_ARGS+=" is_debug=false"
+# Clang or not Clang, that is the question.
+WEE8_BUILD_ARGS+=" is_clang=$$IS_CLANG"
+# Hack to disable bleeding-edge compiler flags.
+WEE8_BUILD_ARGS+=" use_xcode_clang=true"
+# Use local toolchain.
+WEE8_BUILD_ARGS+=" custom_toolchain=\"//build/toolchain/linux/unbundle:default\""
+# Use local stdlibc++ / libc++.
+WEE8_BUILD_ARGS+=" use_custom_libcxx=false"
+# Use local sysroot.
+WEE8_BUILD_ARGS+=" use_sysroot=false"
+# Disable unused GLib2 dependency.
+WEE8_BUILD_ARGS+=" use_glib=false"
+# Expose debug symbols.
+WEE8_BUILD_ARGS+=" v8_expose_symbols=true"
+# Build monolithic library.
+WEE8_BUILD_ARGS+=" is_component_build=false"
+WEE8_BUILD_ARGS+=" v8_enable_i18n_support=false"
+WEE8_BUILD_ARGS+=" v8_enable_gdbjit=false"
+WEE8_BUILD_ARGS+=" v8_use_external_startup_data=false"
+# Disable read-only heap, since it's leaky and HEAPCHECK complains about it.
+# TODO(PiotrSikora): remove when fixed upstream.
+WEE8_BUILD_ARGS+=" v8_enable_shared_ro_heap=false"
+
 # Build wee8.
-third_party/depot_tools/gn gen out/wee8 --args="v8_use_external_startup_data=false v8_enable_i18n_support=false v8_enable_gdbjit=false v8_expose_symbols=true is_component_build=false is_debug=false use_sysroot=false use_custom_libcxx=false"
+third_party/depot_tools/gn gen out/wee8 --args="$$WEE8_BUILD_ARGS"
 third_party/depot_tools/ninja -C out/wee8 wee8
 
 # Move compiled library to the expected destinations.

bazel/external/wee8.patch

@@ -1,34 +1,33 @@
-# 1. Fix handling of f64 globals.
-# 2. Force full GC when destroying VMs.
-# 3. Fix build with -DDEBUG.
---- a/wee8/src/wasm/c-api.cc
-+++ b/wee8/src/wasm/c-api.cc
-@@ -825,7 +825,7 @@ void global_set_f32(v8::Local<v8::Object> global, float val) {
- void global_set_f64(v8::Local<v8::Object> global, double val) {
-   auto v8_object = v8::Utils::OpenHandle<v8::Object, i::JSReceiver>(global);
-   auto v8_global = i::Handle<i::WasmGlobalObject>::cast(v8_object);
--  v8_global->SetF32(val);
-+  v8_global->SetF64(val);
- }
+# 1. Fix linking with unbundled toolchain on macOS.
+# 2. Increase VSZ limit to 4TiB (allows us to start up to 370 VMs).
+--- a/wee8/build/toolchain/gcc_toolchain.gni
++++ b/wee8/build/toolchain/gcc_toolchain.gni
+@@ -355,6 +355,8 @@ template("gcc_toolchain") {
+         # AIX does not support either -D (deterministic output) or response
+         # files.
+         command = "$ar -X64 {{arflags}} -r -c -s {{output}} {{inputs}}"
++      } else if (current_os == "mac") {
++        command = "\"$ar\" {{arflags}} -r -c -s {{output}} {{inputs}}"
+       } else {
+         rspfile = "{{output}}.rsp"
+         rspfile_content = "{{inputs}}"
+@@ -546,7 +548,7 @@ template("gcc_toolchain") {
 
- // Tables
-@@ -1107,7 +1107,7 @@ class StoreImpl {
-   StoreImpl() {}
-
-   ~StoreImpl() {
--#ifdef DEBUG
-+#if 1
-     reinterpret_cast<i::Isolate*>(isolate_)->heap()->PreciseCollectAllGarbage(
-         i::Heap::kNoGCFlags, i::GarbageCollectionReason::kTesting,
-         v8::kGCCallbackFlagForced);
---- a/wee8/third_party/wasm-api/wasm.hh
-+++ b/wee8/third_party/wasm-api/wasm.hh
-@@ -111,7 +111,7 @@ class vec {
-   size_t size_;
-   std::unique_ptr<T[]> data_;
-
--#ifdef DEBUG
-+#if 0
-   void make_data();
-   void free_data();
+       start_group_flag = ""
+       end_group_flag = ""
+-      if (current_os != "aix") {
++      if (current_os != "aix" && current_os != "mac") {
+         # the "--start-group .. --end-group" feature isn't available on the aix ld.
+         start_group_flag = "-Wl,--start-group"
+         end_group_flag = "-Wl,--end-group "
+--- a/wee8/src/wasm/wasm-memory.cc
++++ b/wee8/src/wasm/wasm-memory.cc
+@@ -142,7 +142,7 @@ void* TryAllocateBackingStore(WasmMemoryTracker* memory_tracker, Heap* heap,
+ // address space limits needs to be smaller.
+ constexpr size_t kAddressSpaceLimit = 0x8000000000L;  // 512 GiB
+ #elif V8_TARGET_ARCH_64_BIT
+-constexpr size_t kAddressSpaceLimit = 0x10100000000L;  // 1 TiB + 4 GiB
++constexpr size_t kAddressSpaceLimit = 0x40100000000L;  // 4 TiB + 4 GiB
  #else
+ constexpr size_t kAddressSpaceLimit = 0xC0000000;  // 3 GiB
+ #endif

bazel/repositories.bzl

@@ -120,14 +120,6 @@ def envoy_dependencies(skip_targets = []):
         actual = "@envoy//bazel:boringssl",
     )
 
-    # Binding to an alias pointing to a platform-specific version of wee8.
-    _wee8_linux()
-    _wee8_macos()
-    native.bind(
-        name = "wee8",
-        actual = "@envoy//bazel:wee8",
-    )
-
     # The long repo names (`com_github_fmtlib_fmt` instead of `fmtlib`) are
     # semi-standard in the Bazel community, intended to avoid both duplicate
     # dependencies and name conflicts.
@@ -158,6 +150,7 @@ def envoy_dependencies(skip_targets = []):
     _com_github_curl()
     _com_github_envoyproxy_sqlparser()
     _com_googlesource_quiche()
+    _com_googlesource_chromium_v8()
     _org_llvm_llvm()
     _com_github_wavm_wavm()
     _com_lightstep_tracer_cpp()
@@ -731,26 +724,18 @@ def _com_github_wavm_wavm():
         actual = "@envoy//bazel/foreign_cc:wavm",
     )
 
-def _wee8_linux():
-    location = REPOSITORY_LOCATIONS["wee8_linux"]
+def _com_googlesource_chromium_v8():
+    location = REPOSITORY_LOCATIONS["com_googlesource_chromium_v8"]
     genrule_repository(
-        name = "wee8_linux",
-        urls = location["urls"],
-        sha256 = location["sha256"],
+        name = "com_googlesource_chromium_v8",
         genrule_cmd_file = "@envoy//bazel/external:wee8.genrule_cmd",
         build_file = "@envoy//bazel/external:wee8.BUILD",
         patches = ["@envoy//bazel/external:wee8.patch"],
+        **location
     )
-
-def _wee8_macos():
-    location = REPOSITORY_LOCATIONS["wee8_macos"]
-    genrule_repository(
-        name = "wee8_macos",
-        urls = location["urls"],
-        sha256 = location["sha256"],
-        genrule_cmd_file = "@envoy//bazel/external:wee8.genrule_cmd",
-        build_file = "@envoy//bazel/external:wee8.BUILD",
-        patches = ["@envoy//bazel/external:wee8.patch"],
+    native.bind(
+        name = "wee8",
+        actual = "@com_googlesource_chromium_v8//:wee8",
     )
 
 def _foreign_cc_dependencies():

bazel/repository_locations.bzl

@@ -241,19 +241,11 @@ REPOSITORY_LOCATIONS = dict(
         strip_prefix = "WAVM-95dbf08c8695b8941e7020c557d8612f9d2af895",
         urls = ["https://github.com/WAVM/WAVM/archive/95dbf08c8695b8941e7020c557d8612f9d2af895.tar.gz"],
     ),
-    wee8_linux = dict(
+    com_googlesource_chromium_v8 = dict(
         # This archive was created using https://storage.googleapis.com/envoyproxy-wee8/wee8-archive.sh
-        # and contains complete checkout of v8 with all dependencies necessary to build wee8 on Linux-x86_64.
-        # TODO(PiotrSikora): switch to local compiler and provide single platform-agnostic archive.
-        sha256 = "1caebced30cb9d3531be4720b70c9132c988b362160f7721bc01caeb572c0eb7",
-        urls = ["https://storage.googleapis.com/envoyproxy-wee8/wee8-7.5.288.22-linux-x86_64.tar.gz"],
-    ),
-    wee8_macos = dict(
-        # This archive was created using https://storage.googleapis.com/envoyproxy-wee8/wee8-archive.sh
-        # and contains complete checkout of v8 with all dependencies necessary to build wee8 on macOS-x86_64.
-        # TODO(PiotrSikora): switch to local compiler and provide single platform-agnostic archive.
-        sha256 = "f84e423417db0b03b96e853b7d92f69be7d4936a33d1e8e05848fb146925ff68",
-        urls = ["https://storage.googleapis.com/envoyproxy-wee8/wee8-7.5.288.22-macos-x86_64.tar.gz"],
+        # and contains complete checkout of v8 with all dependencies necessary to build wee8.
+        sha256 = "54d42b0de3055b5ddc6efc0ee1cf7fd2ef77696e90391129f403edc345511e1f",
+        urls = ["https://storage.googleapis.com/envoyproxy-wee8/wee8-7.8.196.tar.gz"],
     ),
     io_opencensus_cpp = dict(
         sha256 = "8d6016e47c2e19e7acbadb6f905b8c422748c64299d71101ac8f28151677e195",

ci/build_container/build_container_centos.sh

@@ -6,8 +6,10 @@ set -e
 yum install -y centos-release-scl epel-release
 yum update -y
 yum install -y devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-binutils java-1.8.0-openjdk-headless rsync \
-    rh-git218 wget unzip which make patch devtoolset-7-libatomic-devel openssl python27 \
-    libtool autoconf tcpdump glib2-devel
+    rh-git218 wget unzip which make patch ninja-build devtoolset-7-libatomic-devel openssl python27 \
+    libtool autoconf tcpdump
+
+ln -s /usr/bin/ninja-build /usr/bin/ninja
 
 # SLES 11 has older glibc than CentOS 7, so pre-built binary for it works on CentOS 7
 LLVM_VERSION=8.0.0

ci/build_container/build_container_common.sh

@@ -21,12 +21,4 @@ if [[ "$(uname -m)" == "x86_64" ]]; then
   curl -sLo cmake-"$VERSION".tar.gz https://github.com/Kitware/CMake/releases/download/v"$VERSION"/cmake-"$VERSION"-Linux-x86_64.tar.gz \
     && echo "$SHA256" cmake-"$VERSION".tar.gz | sha256sum --check \
     && tar -zxf cmake-"$VERSION".tar.gz -C /usr --strip-components=1
-
-  # ninja
-  VERSION=1.8.2
-  SHA256=d2fea9ff33b3ef353161ed906f260d565ca55b8ca0568fa07b1d2cab90a84a07
-  curl -sLo ninja-"$VERSION".zip https://github.com/ninja-build/ninja/releases/download/v"$VERSION"/ninja-linux.zip \
-    && echo "$SHA256" ninja-"$VERSION".zip | sha256sum --check \
-    && unzip ninja-"$VERSION".zip \
-    && mv ninja /usr/bin
 fi

ci/build_container/build_container_ubuntu.sh

@@ -19,7 +19,7 @@ update-alternatives --config gcc
 update-alternatives --config g++
 
 apt-get install -y --no-install-recommends curl wget make git python python-pip python-setuptools python3 python3-pip \
-  unzip bc libtool automake zip time gdb strace tshark tcpdump patch xz-utils rsync ssh-client libglib2.0-dev
+  unzip bc libtool ninja-build automake zip time gdb strace tshark tcpdump patch xz-utils rsync ssh-client
 
 # clang 8.
 case $ARCH in