access log: add JSON logging mode

api/envoy/config/accesslog/v2/file.proto

@@ -4,6 +4,7 @@ package envoy.config.accesslog.v2;
 option go_package = "v2";
 
 import "validate/validate.proto";
+import "google/protobuf/struct.proto";
 
 // [#protodoc-title: File access log]
 
@@ -17,5 +18,11 @@ message FileAccessLog {
   // Access log format. Envoy supports :ref:`custom access log formats
   // <config_access_log_format>` as well as a :ref:`default format
   // <config_access_log_default_format>`.
-  string format = 2;
+  oneof access_log_format {
+    // Access log :ref:`format string<config_access_log_format_strings>`
+    string format = 2;
+
+    // Access log :ref:`format dictionary<config_access_log_format_dictionaries>`
+    google.protobuf.Struct json_format = 3;
+  }
 }

docs/root/configuration/access_log.rst

@@ -17,16 +17,91 @@ Access logs are configured as part of the :ref:`HTTP connection manager config
 Format rules
 ------------
 
-The access log format string contains either command operators or other characters interpreted as a
-plain string. The access log formatter does not make any assumptions about a new line separator, so one
+Access log formats contain command operators that extract the relevant data and insert it.
+They support two formats: :ref:`"format strings" <config_access_log_format_strings>` and
+:ref:`"format dictionaries" <config_access_log_format_dictionaries>`. In both cases, the command operators
+are used to extract the relevant data, which is then inserted into the specified log format.
+Only one access log format may be specified at a time.
+
+.. _config_access_log_format_strings:
+
+Format Strings
+--------------
+
+Format strings are plain strings, specified using the ``format`` key. They may contain
+either command operators or other characters interpreted as a plain string. 
+The access log formatter does not make any assumptions about a new line separator, so one
 has to specified as part of the format string.
 See the :ref:`default format <config_access_log_default_format>` for an example.
-Note that the access log line will contain a '-' character for every not set/empty value.
 
-The same format strings are used by different types of access logs (such as HTTP and TCP). Some
+.. _config_access_log_default_format:
+
+Default Format String
+---------------------
+
+If custom format string is not specified, Envoy uses the following default format:
+
+.. code-block:: none
+
+  [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%"
+  %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION%
+  %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%"
+  "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n
+
+Example of the default Envoy access log format:
+
+.. code-block:: none
+
+  [2016-04-15T20:17:00.310Z] "POST /api/v1/locations HTTP/2" 204 - 154 0 226 100 "10.0.35.28"
+  "nsq2http" "cc21d9b0-cf5c-432b-8c7e-98aeb7988cd2" "locations" "tcp://10.0.2.1:80"
+
+.. _config_access_log_format_dictionaries:
+
+Format Dictionaries
+-------------------
+
+Format dictionaries are dictionaries that specify a structured access log output format,
+specified using the ``json_format`` key. This allows logs to be output in a structured format
+such as JSON.
+Similar to format strings, command operators are evaluated and their values inserted into the format
+dictionary to construct the log output.
+
+For example, with the following format provided in the configuration:
+
+.. code-block:: json
+
+  {
+    "config": {
+      "json_format": {
+          "protocol": "%PROTOCOL%",
+          "duration": "%DURATION%",
+          "my_custom_header": "%REQ(MY_CUSTOM_HEADER)%"
+      }
+    }
+  }
+
+The following JSON object would be written to the log file:
+
+.. code-block:: json
+
+  {"protocol": "HTTP/1.1", "duration": "123", "my_custom_header": "value_of_MY_CUSTOM_HEADER"}
+
+This allows you to specify a custom key for each command operator.
+
+Format dictionaries have the following restrictions:
+
+* The dictionary must map strings to strings (specifically, strings to command operators). Nesting is not currently supported.
+
+Command Operators
+-----------------
+
+Command operators are used to extract values that will be inserted into the access logs.
+The same operators are used by different types of access logs (such as HTTP and TCP). Some
 fields may have slightly different meanings, depending on what type of log it is. Differences
 are noted.
 
+Note that if a value is not set/empty, the logs will contain a '-' character.
+
 The following command operators are supported:
 
 .. _config_access_log_format_start_time:
@@ -232,24 +307,3 @@ The following command operators are supported:
     String value set on ssl connection socket for Server Name Indication (SNI)
   TCP
     String value set on ssl connection socket for Server Name Indication (SNI)
-
-.. _config_access_log_default_format:
-
-Default format
---------------
-
-If custom format is not specified, Envoy uses the following default format:
-
-.. code-block:: none
-
-  [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%"
-  %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION%
-  %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%"
-  "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n
-
-Example of the default Envoy access log format:
-
-.. code-block:: none
-
-  [2016-04-15T20:17:00.310Z] "POST /api/v1/locations HTTP/2" 204 - 154 0 226 100 "10.0.35.28"
-  "nsq2http" "cc21d9b0-cf5c-432b-8c7e-98aeb7988cd2" "locations" "tcp://10.0.2.1:80"

include/envoy/access_log/access_log.h

@@ -71,11 +71,20 @@ typedef std::shared_ptr<Instance> InstanceSharedPtr;
 
 /**
  * Interface for access log formatter.
+ * Formatters provide a complete access log output line for the given headers/trailers/stream.
  */
 class Formatter {
 public:
   virtual ~Formatter() {}
 
+  /**
+   * Return a formatted access log line.
+   * @param request_headers supplies the request headers.
+   * @param response_headers supplies the response headers.
+   * @param response_trailers supplies the response trailers.
+   * @param stream_info supplies the stream info.
+   * @return std::string string containing the complete formatted access log line.
+   */
   virtual std::string format(const Http::HeaderMap& request_headers,
                              const Http::HeaderMap& response_headers,
                              const Http::HeaderMap& response_trailers,
@@ -84,5 +93,29 @@ class Formatter {
 
 typedef std::unique_ptr<Formatter> FormatterPtr;
 
+/**
+ * Interface for access log provider.
+ * FormatterProviders extract information from the given headers/trailers/stream.
+ */
+class FormatterProvider {
+public:
+  virtual ~FormatterProvider() {}
+
+  /**
+   * Extract a value from the provided headers/trailers/stream.
+   * @param request_headers supplies the request headers.
+   * @param response_headers supplies the response headers.
+   * @param response_trailers supplies the response trailers.
+   * @param stream_info supplies the stream info.
+   * @return std::string containing a single value extracted from the given headers/trailers/stream.
+   */
+  virtual std::string format(const Http::HeaderMap& request_headers,
+                             const Http::HeaderMap& response_headers,
+                             const Http::HeaderMap& response_trailers,
+                             const StreamInfo::StreamInfo& stream_info) const PURE;
+};
+
+typedef std::unique_ptr<FormatterProvider> FormatterProviderPtr;
+
 } // namespace AccessLog
 } // namespace Envoy

source/common/access_log/access_log_formatter.cc

@@ -54,7 +54,7 @@ AccessLogFormatUtils::protocolToString(const absl::optional<Http::Protocol>& pro
 }
 
 FormatterImpl::FormatterImpl(const std::string& format) {
-  formatters_ = AccessLogFormatParser::parse(format);
+  providers_ = AccessLogFormatParser::parse(format);
 }
 
 std::string FormatterImpl::format(const Http::HeaderMap& request_headers,
@@ -64,14 +64,54 @@ std::string FormatterImpl::format(const Http::HeaderMap& request_headers,
   std::string log_line;
   log_line.reserve(256);
 
-  for (const FormatterPtr& formatter : formatters_) {
-    log_line +=
-        formatter->format(request_headers, response_headers, response_trailers, stream_info);
+  for (const FormatterProviderPtr& provider : providers_) {
+    log_line += provider->format(request_headers, response_headers, response_trailers, stream_info);
   }
 
   return log_line;
 }
 
+JsonFormatterImpl::JsonFormatterImpl(std::unordered_map<std::string, std::string>& format_mapping) {
+  for (const auto& pair : format_mapping) {
+    auto providers = AccessLogFormatParser::parse(pair.second);
+    json_output_format_.emplace(pair.first, FormatterPtr{new FormatterImpl(pair.second)});
+  }
+}
+
+std::string JsonFormatterImpl::format(const Http::HeaderMap& request_headers,
+                                      const Http::HeaderMap& response_headers,
+                                      const Http::HeaderMap& response_trailers,
+                                      const StreamInfo::StreamInfo& stream_info) const {
+  const auto output_map = toMap(request_headers, response_headers, response_trailers, stream_info);
+
+  ProtobufWkt::Struct output_struct;
+  for (const auto& pair : output_map) {
+    ProtobufWkt::Value string_value;
+    string_value.set_string_value(pair.second);
+    (*output_struct.mutable_fields())[pair.first] = string_value;
+  }
+
+  std::string log_line;
+  const auto conversion_status = ProtobufUtil::MessageToJsonString(output_struct, &log_line);
+  if (!conversion_status.ok()) {
+    log_line =
+        fmt::format("Error serializing access log to JSON: {}", conversion_status.ToString());
+  }
+
+  return log_line;
+}
+
+std::unordered_map<std::string, std::string> JsonFormatterImpl::toMap(
+    const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers,
+    const Http::HeaderMap& response_trailers, const StreamInfo::StreamInfo& stream_info) const {
+  std::unordered_map<std::string, std::string> output;
+  for (const auto& pair : json_output_format_) {
+    output.emplace(pair.first, pair.second->format(request_headers, response_headers,
+                                                   response_trailers, stream_info));
+  }
+  return output;
+}
+
 void AccessLogFormatParser::parseCommandHeader(const std::string& token, const size_t start,
                                                std::string& main_header,
                                                std::string& alternative_header,
@@ -131,16 +171,16 @@ void AccessLogFormatParser::parseCommand(const std::string& token, const size_t
 }
 
 // TODO(derekargueta): #2967 - Rewrite AccessLogformatter with parser library & formal grammar
-std::vector<FormatterPtr> AccessLogFormatParser::parse(const std::string& format) {
+std::vector<FormatterProviderPtr> AccessLogFormatParser::parse(const std::string& format) {
   std::string current_token;
-  std::vector<FormatterPtr> formatters;
+  std::vector<FormatterProviderPtr> formatters;
   const std::string DYNAMIC_META_TOKEN = "DYNAMIC_METADATA(";
   const std::regex command_w_args_regex(R"EOF(%([A-Z]|_)+(\([^\)]*\))?(:[0-9]+)?(%))EOF");
 
   for (size_t pos = 0; pos < format.length(); ++pos) {
     if (format[pos] == '%') {
       if (!current_token.empty()) {
-        formatters.emplace_back(new PlainStringFormatter(current_token));
+        formatters.emplace_back(FormatterProviderPtr{new PlainStringFormatter(current_token)});
         current_token = "";
       }
 
@@ -163,42 +203,43 @@ std::vector<FormatterPtr> AccessLogFormatParser::parse(const std::string& format
 
         parseCommandHeader(token, ReqParamStart, main_header, alternative_header, max_length);
 
-        formatters.emplace_back(
-            new RequestHeaderFormatter(main_header, alternative_header, max_length));
+        formatters.emplace_back(FormatterProviderPtr{
+            new RequestHeaderFormatter(main_header, alternative_header, max_length)});
       } else if (token.find("RESP(") == 0) {
         std::string main_header, alternative_header;
         absl::optional<size_t> max_length;
 
         parseCommandHeader(token, RespParamStart, main_header, alternative_header, max_length);
 
-        formatters.emplace_back(
-            new ResponseHeaderFormatter(main_header, alternative_header, max_length));
+        formatters.emplace_back(FormatterProviderPtr{
+            new ResponseHeaderFormatter(main_header, alternative_header, max_length)});
       } else if (token.find("TRAILER(") == 0) {
         std::string main_header, alternative_header;
         absl::optional<size_t> max_length;
 
         parseCommandHeader(token, TrailParamStart, main_header, alternative_header, max_length);
 
-        formatters.emplace_back(
-            new ResponseTrailerFormatter(main_header, alternative_header, max_length));
+        formatters.emplace_back(FormatterProviderPtr{
+            new ResponseTrailerFormatter(main_header, alternative_header, max_length)});
       } else if (token.find(DYNAMIC_META_TOKEN) == 0) {
         std::string filter_namespace;
         absl::optional<size_t> max_length;
         std::vector<std::string> path;
         const size_t start = DYNAMIC_META_TOKEN.size();
 
         parseCommand(token, start, ":", filter_namespace, path, max_length);
-        formatters.emplace_back(new DynamicMetadataFormatter(filter_namespace, path, max_length));
+        formatters.emplace_back(
+            FormatterProviderPtr{new DynamicMetadataFormatter(filter_namespace, path, max_length)});
       } else if (token.find("START_TIME") == 0) {
         const size_t parameters_length = pos + StartTimeParamStart + 1;
         const size_t parameters_end = command_end_position - parameters_length;
 
         const std::string args = token[StartTimeParamStart - 1] == '('
                                      ? token.substr(StartTimeParamStart, parameters_end)
                                      : "";
-        formatters.emplace_back(new StartTimeFormatter(args));
+        formatters.emplace_back(FormatterProviderPtr{new StartTimeFormatter(args)});
       } else {
-        formatters.emplace_back(new StreamInfoFormatter(token));
+        formatters.emplace_back(FormatterProviderPtr{new StreamInfoFormatter(token)});
       }
       pos = command_end_position;
     } else {
@@ -207,7 +248,7 @@ std::vector<FormatterPtr> AccessLogFormatParser::parse(const std::string& format
   }
 
   if (!current_token.empty()) {
-    formatters.emplace_back(new PlainStringFormatter(current_token));
+    formatters.emplace_back(FormatterProviderPtr{new PlainStringFormatter(current_token)});
   }
 
   return formatters;