Mysql proxy filter (#4975)

This filter contains the logic to decode the mysql wire protocol and SQL queries (SQL99 only). The code is based on our internal version at VMware. The SQL parser can be found at https://github.com/rshriram/sql-parser. Its a cleaned up version of Hyrise SQL parser. I am keeping the code as a separate library as importing the sources into envoy will cause a lot of changes to the code. Signed-off-by: Giorgio Valentini <[email protected]> Signed-off-by: Deepa Kalani <[email protected]> Signed-off-by: Shriram Rajagopalan <[email protected]> Signed-off-by: Venil Noronha <[email protected]>
istio · Jan 17, 2019 · b3be571 · b3be571
1 parent 0610c74
commit b3be571
Show file tree

Hide file tree

Showing 41 changed files with 5,161 additions and 1 deletion.
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -18,3 +18,5 @@
 /*/extensions/filters/network/sni_cluster @rshriram @lizan
 # tracers.datadog extension
 /*/extensions/tracers/datadog @cgilmour @palazzem
+# mysql_proxy extension
+/*/extensions/filters/network/mysql_proxy @rshriram @venilnoronha @mattklein123
diff --git a/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD b/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD
@@ -0,0 +1,8 @@
+load("//bazel:api_build_system.bzl", "api_proto_library_internal")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library_internal(
+    name = "mysql_proxy",
+    srcs = ["mysql_proxy.proto"],
+)
diff --git a/api/envoy/config/filter/network/mysql_proxy/v1alpha1/mysql_proxy.proto b/api/envoy/config/filter/network/mysql_proxy/v1alpha1/mysql_proxy.proto
@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package envoy.config.filter.network.mysql_proxy.v1alpha1;
+option java_package = "io.envoyproxy.envoy.config.filter.network.mysql_proxy.v1alpha1";
+option java_multiple_files = true;
+option go_package = "v1alpha1";
+
+import "validate/validate.proto";
+
+// [#protodoc-title: MySQL proxy]
+// MySQL Proxy :ref:`configuration overview <config_network_filters_mysql_proxy>`.
+message MySQLProxy {
+  // The human readable prefix to use when emitting :ref:`statistics
+  // <config_network_filters_mysql_proxy_stats>`.
+  string stat_prefix = 1 [(validate.rules).string.min_bytes = 1];
+
+  // [#not-implemented-hide:] The optional path to use for writing MySQL access logs.
+  // If the access log field is empty, access logs will not be written.
+  string access_log = 2;
+}
diff --git a/bazel/external/sqlparser.BUILD b/bazel/external/sqlparser.BUILD
@@ -0,0 +1,11 @@
+licenses(["notice"])  # Apache 2
+
+cc_library(
+    name = "sqlparser",
+    srcs = glob(["src/**/*.cpp"]),
+    hdrs = glob([
+        "include/**/*.h",
+        "src/**/*.h",
+    ]),
+    visibility = ["//visibility:public"],
+)
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -297,6 +297,7 @@ def envoy_dependencies(path = "@envoy_deps//", skip_targets = []):
     _com_github_tencent_rapidjson()
     _com_google_googletest()
     _com_google_protobuf()
+    _com_github_envoyproxy_sqlparser()
 
     # Used for bundling gcovr into a relocatable .par file.
     _repository_impl("subpar")
@@ -349,6 +350,16 @@ def _com_github_cyan4973_xxhash():
         actual = "@com_github_cyan4973_xxhash//:xxhash",
     )
 
+def _com_github_envoyproxy_sqlparser():
+    _repository_impl(
+        name = "com_github_envoyproxy_sqlparser",
+        build_file = "@envoy//bazel/external:sqlparser.BUILD",
+    )
+    native.bind(
+        name = "sqlparser",
+        actual = "@com_github_envoyproxy_sqlparser//:sqlparser",
+    )
+
 def _com_github_eile_tclap():
     _repository_impl(
         name = "com_github_eile_tclap",

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
@@ -42,6 +42,11 @@ REPOSITORY_LOCATIONS = dict(
         strip_prefix = "xxHash-0.6.5",
         urls = ["https://github.com/Cyan4973/xxHash/archive/v0.6.5.tar.gz"],
     ),
+    com_github_envoyproxy_sqlparser = dict(
+        sha256 = "425dfee0c4fe9aff8acf2365cde3dd2ba7fb878d2ba37562d33920e34c40c05e",
+        strip_prefix = "sql-parser-5f50c68bdf5f107692bb027d1c568f67597f4d7f",
+        urls = ["https://github.com/envoyproxy/sql-parser/archive/5f50c68bdf5f107692bb027d1c568f67597f4d7f.tar.gz"],
+    ),
     com_github_eile_tclap = dict(
         sha256 = "f0ede0721dddbb5eba3a47385a6e8681b14f155e1129dd39d1a959411935098f",
         strip_prefix = "tclap-tclap-1-2-1-release-final",

diff --git a/docs/root/configuration/network_filters/mysql_proxy_filter.rst b/docs/root/configuration/network_filters/mysql_proxy_filter.rst
@@ -0,0 +1,115 @@
+.. _config_network_filters_mysql_proxy:
+
+MySQL proxy
+===========
+
+The MySQL proxy filter decodes the wire protocol between the MySQL client
+and server. It decodes the SQL queries in the payload (SQL99 format only).
+The decoded info is emitted as dynamic metadata that can be combined with
+access log filters to get detailed information on tables accessed as well
+as operations performed on each table.
+
+.. attention::
+
+   The mysql_proxy filter is experimental and is currently under active
+   development. Capabilities will be expanded over time and the
+   configuration structures are likely to change.
+
+.. _config_network_filters_mysql_proxy_config:
+
+Configuration
+-------------
+
+The MySQL proxy filter should be chained with the TCP proxy filter as shown
+in the configuration snippet below:
+
+.. code-block:: yaml
+
+  filter_chains:
+  - filters:
+    - name: envoy.filters.network.mysql_proxy
+      config:
+        stat_prefix: mysql
+    - name: envoy.tcp_proxy
+      config:
+        stat_prefix: tcp
+        cluster: ...
+
+
+.. _config_network_filters_mysql_proxy_stats:
+
+Statistics
+----------
+
+Every configured MySQL proxy filter has statistics rooted at *mysql.<stat_prefix>.* with the
+following statistics:
+
+.. csv-table::
+  :header: Name, Type, Description
+  :widths: 1, 1, 2
+
+  auth_switch_request, Counter, Number of times the upstream server requested clients to switch to a different authentication method
+  decoder_errors, Counter, Number of MySQL protocol decoding errors
+  login_attempts, Counter, Number of login attempts
+  login_failures, Counter, Number of login failures
+  protocol_errors, Counter, Number of out of sequence protocol messages encountered in a session
+  queries_parse_error, Counter, Number of MySQL queries parsed with errors
+  queries_parsed, Counter, Number of MySQL queries successfully parsed
+  sessions, Counter, Number of MySQL sessions since start
+  upgraded_to_ssl, Counter, Number of sessions/connections that were upgraded to SSL
+
+.. _config_network_filters_mysql_proxy_dynamic_metadata:
+
+Dynamic Metadata
+----------------
+
+The MySQL filter emits the following dynamic metadata for each SQL query parsed:
+
+.. csv-table::
+  :header: Name, Type, Description
+  :widths: 1, 1, 2
+
+  <table.db>, string, The resource name in *table.db* format. The resource name defaults to the table being accessed if the database cannot be inferred.
+  [], list, A list of strings representing the operations executed on the resource. Operations can be one of insert/update/select/drop/delete/create/alter/show.
+
+.. _config_network_filters_mysql_proxy_rbac:
+
+RBAC Enforcement on Table Accesses
+----------------------------------
+
+The dynamic metadata emitted by the MySQL filter can be used in conjunction
+with the RBAC filter to control accesses to individual tables in a
+database. The following configuration snippet shows an example RBAC filter
+configuration that denies SQL queries with _update_ statements to the
+_catalog_ table in the _productdb_ database.
+
+.. code-block:: yaml
+
+  filter_chains:
+  - filters:
+    - name: envoy.filters.network.mysql_proxy
+      config:
+        stat_prefix: mysql
+    - name: envoy.filters.network.rbac
+      config:
+        stat_prefix: rbac
+        rules:
+          action: DENY
+          policies:
+            "product-viewer":
+              permissions:
+              - metadata:
+                  filter: envoy.filters.network.mysql_proxy
+                  path:
+                  - key: catalog.productdb
+                  value:
+                    list_match:
+                      one_of:
+                        string_match:
+                          exact: update
+              principals:
+              - any: true
+    - name: envoy.tcp_proxy
+      config:
+        stat_prefix: tcp
+        cluster: mysql
diff --git a/docs/root/configuration/network_filters/network_filters.rst b/docs/root/configuration/network_filters/network_filters.rst
@@ -14,6 +14,7 @@ filters.
   echo_filter
   ext_authz_filter
   mongo_proxy_filter
+  mysql_proxy_filter
   rate_limit_filter
   rbac_filter
   redis_proxy_filter

diff --git a/docs/root/configuration/well_known_dynamic_metadata.rst b/docs/root/configuration/well_known_dynamic_metadata.rst
@@ -8,11 +8,12 @@ Filters can emit dynamic metadata via the *setDynamicMetadata* routine in the
 :repo:`Connection <include/envoy/network/connection.h>`. This metadata emitted by a filter can be
 consumed by other filters and useful features can be built by stacking such filters. For example,
 a logging filter can consume dynamic metadata from an RBAC filter to log details about runtime
-shadow rule behavior. Another example is where an RBAC filter permits/restricts MongoDB operations
+shadow rule behavior. Another example is where an RBAC filter permits/restricts MySQL/MongoDB operations
 by looking at the operational metadata emitted by the MongoDB filter.
 
 The following Envoy filters emit dynamic metadata that other filters can leverage.
 
 * :ref:`Mongo Proxy Filter <config_network_filters_mongo_proxy_dynamic_metadata>`
+* :ref:`MySQL Proxy Filter <config_network_filters_mysql_proxy_dynamic_metadata>`
 * :ref:`Role Based Access Control (RBAC) Filter <config_http_filters_rbac_dynamic_metadata>`
 * :ref:`Role Based Access Control (RBAC) Network Filter <config_network_filters_rbac_dynamic_metadata>`
diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst
@@ -18,6 +18,7 @@ Version history
   <envoy_api_field_config.filter.http.ext_authz.v2alpha.AuthorizationResponse.allowed_upstream_headers>` replaces the previous *allowed_authorization_headers* object. 
   All the control header lists now support :ref:`string matcher <envoy_api_msg_type.matcher.StringMatcher>` instead of standard string.
 * http: added new grpc_http1_reverse_bridge filter for converting gRPC requests into HTTP/1.1 requests.
+* mysql: added a MySQL proxy filter that is capable of parsing SQL queries over MySQL wire protocol. Refer to ::ref:`MySQL proxy<config_network_filters_mysql_proxy>` for more details.
 * redis: added :ref:`success and error stats <config_network_filters_redis_proxy_per_command_stats>` for commands.
 * router: added ability to configure a :ref:`retry policy <envoy_api_msg_route.RetryPolicy>` at the
   virtual host level.

diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl
@@ -69,6 +69,7 @@ EXTENSIONS = {
     "envoy.filters.network.ext_authz":                  "//source/extensions/filters/network/ext_authz:config",
     "envoy.filters.network.http_connection_manager":    "//source/extensions/filters/network/http_connection_manager:config",
     "envoy.filters.network.mongo_proxy":                "//source/extensions/filters/network/mongo_proxy:config",
+    "envoy.filters.network.mysql_proxy":                "//source/extensions/filters/network/mysql_proxy:config",
     "envoy.filters.network.ratelimit":                  "//source/extensions/filters/network/ratelimit:config",
     "envoy.filters.network.rbac":                       "//source/extensions/filters/network/rbac:config",
     "envoy.filters.network.redis_proxy":                "//source/extensions/filters/network/redis_proxy:config",
@@ -186,6 +187,7 @@ WINDOWS_EXTENSIONS = {
     #"envoy.filters.network.ext_authz":                  "//source/extensions/filters/network/ext_authz:config",
     #"envoy.filters.network.http_connection_manager":    "//source/extensions/filters/network/http_connection_manager:config",
     #"envoy.filters.network.mongo_proxy":                "//source/extensions/filters/network/mongo_proxy:config",
+    #"envoy.filters.network.mysql_proxy":                "//source/extensions/filters/network/mysql_proxy:config",
     #"envoy.filters.network.redis_proxy":                "//source/extensions/filters/network/redis_proxy:config",
     #"envoy.filters.network.ratelimit":                  "//source/extensions/filters/network/ratelimit:config",
     "envoy.filters.network.tcp_proxy":                  "//source/extensions/filters/network/tcp_proxy:config",

diff --git a/source/extensions/filters/network/mysql_proxy/BUILD b/source/extensions/filters/network/mysql_proxy/BUILD
@@ -0,0 +1,60 @@
+licenses(["notice"])  # Apache 2
+
+# MySQL proxy L7 network filter.
+# Public docs: docs/root/configuration/network_filters/mysql_proxy_filter.rst
+
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_library",
+    "envoy_package",
+)
+
+envoy_package()
+
+envoy_cc_library(
+    name = "proxy_lib",
+    srcs = [
+        "mysql_codec_clogin.cc",
+        "mysql_codec_clogin_resp.cc",
+        "mysql_codec_command.cc",
+        "mysql_codec_greeting.cc",
+        "mysql_codec_switch_resp.cc",
+        "mysql_decoder.cc",
+        "mysql_filter.cc",
+        "mysql_utils.cc",
+    ],
+    hdrs = [
+        "mysql_codec.h",
+        "mysql_codec_clogin.h",
+        "mysql_codec_clogin_resp.h",
+        "mysql_codec_command.h",
+        "mysql_codec_greeting.h",
+        "mysql_codec_switch_resp.h",
+        "mysql_decoder.h",
+        "mysql_filter.h",
+        "mysql_session.h",
+        "mysql_utils.h",
+    ],
+    external_deps = ["sqlparser"],
+    deps = [
+        "//include/envoy/network:filter_interface",
+        "//include/envoy/server:filter_config_interface",
+        "//include/envoy/stats:stats_interface",
+        "//include/envoy/stats:stats_macros",
+        "//source/common/config:filter_json_lib",
+        "//source/common/network:filter_lib",
+        "//source/extensions/filters/network:well_known_names",
+    ],
+)
+
+envoy_cc_library(
+    name = "config",
+    srcs = ["mysql_config.cc"],
+    hdrs = ["mysql_config.h"],
+    deps = [
+        ":proxy_lib",
+        "//source/extensions/filters/network:well_known_names",
+        "//source/extensions/filters/network/common:factory_base_lib",
+        "@envoy_api//envoy/config/filter/network/mysql_proxy/v1alpha1:mysql_proxy_cc",
+    ],
+)