Serialize MDC under top level

README.md

@@ -89,9 +89,16 @@ We recommend using this library to log into a JSON log file and let Filebeat sen
 |[`error.message`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getMessage()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getMessage())|
 |[`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html)|[`Throwable#getStackTrace()`](https://docs.oracle.com/javase/7/docs/api/java/lang/Throwable.html#getStackTrace())|
 |[`process.thread.name`](https://www.elastic.co/guide/en/ecs/current/ecs-process.html)|[`LogEvent#getThreadName()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getThreadName()) |
-|[`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())|
+|Each MDC entry is a top-level field <a href="#note1" id="note1ref"><sup>1</sup></a>|[`LogEvent#getContextMap()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextMap())|
 |[`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html)|[`LogEvent#getContextStack()`](https://logging.apache.org/log4j/log4j-2.3/log4j-core/apidocs/org/apache/logging/log4j/core/LogEvent.html#getContextStack())|
 
+<a id="note1" href="#note1ref"><sup>1</sup></a> It's recommended to use existing [ECS fields](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for MDC values.
+
+If there is no appropriate ECS field,
+consider prefixing your fields with `labels.`, as in `labels.foo`, for simple key/value pairs.
+For nested structures consider prefixing with `custom.` to make sure you won't get conflicts if ECS later adds the same fields but with a different mapping.
+
+
 ## Getting Started
 
 ### Step 1: Configure application logging

ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java

@@ -26,14 +26,10 @@
 
 import java.io.PrintWriter;
 import java.io.Writer;
-import java.util.Arrays;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 public class EcsJsonSerializer {
 
-    public static final List<String> DEFAULT_TOP_LEVEL_LABELS = Arrays.asList("trace.id", "transaction.id", "span.id", "error.id", "service.name");
     private static final TimestampSerializer TIMESTAMP_SERIALIZER = new TimestampSerializer();
     private static final ThreadLocal<StringBuilder> messageStringBuilder = new ThreadLocal<StringBuilder>();
     private static final  String NEW_LINE = System.getProperty("line.separator");
@@ -144,14 +140,11 @@ public static void serializeOrigin(StringBuilder builder, String fileName, Strin
         builder.append("},");
     }
 
-    public static void serializeLabels(StringBuilder builder, Map<String, ?> labels, Set<String> topLevelLabels) {
-        if (!labels.isEmpty()) {
-            for (Map.Entry<String, ?> entry : labels.entrySet()) {
+    public static void serializeMDC(StringBuilder builder, Map<String, ?> properties) {
+        if (!properties.isEmpty()) {
+            for (Map.Entry<String, ?> entry : properties.entrySet()) {
                 builder.append('\"');
                 String key = entry.getKey();
-                if (!topLevelLabels.contains(key)) {
-                    builder.append("labels.");
-                }
                 JsonUtils.quoteAsString(key, builder);
                 builder.append("\":\"");
                 JsonUtils.quoteAsString(toNullSafeString(String.valueOf(entry.getValue())), builder);

ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java

@@ -63,7 +63,7 @@ void testSimpleLog() throws Exception {
     void testThreadContext() throws Exception {
         putMdc("foo", "bar");
         debug("test");
-        assertThat(getLastLogLine().get("labels.foo").textValue()).isEqualTo("bar");
+        assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
     }
 
     @Test
@@ -75,21 +75,15 @@ void testThreadContextStack() throws Exception {
     }
 
     @Test
-    void testTopLevelLabels() throws Exception {
+    void testMdc() throws Exception {
         putMdc("transaction.id", "0af7651916cd43dd8448eb211c80319c");
         putMdc("span.id", "foo");
+        putMdc("foo", "bar");
         debug("test");
         assertThat(getLastLogLine().get("labels.transaction.id")).isNull();
         assertThat(getLastLogLine().get("transaction.id").textValue()).isEqualTo("0af7651916cd43dd8448eb211c80319c");
         assertThat(getLastLogLine().get("span.id").textValue()).isEqualTo("foo");
-    }
-
-    @Test
-    void testCustomTopLevelLabels() throws Exception {
-        putMdc("top_level", "foo");
-        debug("test");
-        assertThat(getLastLogLine().get("labels.top_level")).isNull();
-        assertThat(getLastLogLine().get("top_level").textValue()).isEqualTo("foo");
+        assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
     }
 
     @Test

log4j-ecs-layout/src/main/java/co/elastic/logging/log4j/EcsLayout.java

@@ -30,14 +30,10 @@
 import org.apache.log4j.spi.LoggingEvent;
 import org.apache.log4j.spi.ThrowableInformation;
 
-import java.util.HashSet;
-import java.util.Set;
-
 public class EcsLayout extends Layout {
 
     private boolean stackTraceAsArray = false;
     private String serviceName;
-    private Set<String> topLevelLabels = new HashSet<String>(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS);
     private boolean includeOrigin;
     private String eventDataset;
 
@@ -51,7 +47,7 @@ public String format(LoggingEvent event) {
         EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
         EcsJsonSerializer.serializeThreadName(builder, event.getThreadName());
         EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName());
-        EcsJsonSerializer.serializeLabels(builder, event.getProperties(), topLevelLabels);
+        EcsJsonSerializer.serializeMDC(builder, event.getProperties());
         EcsJsonSerializer.serializeTag(builder, event.getNDC());
         if (includeOrigin) {
             LocationInfo locationInformation = event.getLocationInformation();
@@ -102,10 +98,6 @@ public void setStackTraceAsArray(boolean stackTraceAsArray) {
         this.stackTraceAsArray = stackTraceAsArray;
     }
 
-    public void addTopLevelLabel(String topLevelLabel) {
-        this.topLevelLabels.add(topLevelLabel);
-    }
-
     public void setEventDataset(String eventDataset) {
         this.eventDataset = eventDataset;
     }

log4j-ecs-layout/src/test/java/co/elastic/logging/log4j/Log4jEcsLayoutTest.java

@@ -52,7 +52,6 @@ void setUp() {
         ecsLayout.setServiceName("test");
         ecsLayout.setStackTraceAsArray(true);
         ecsLayout.setIncludeOrigin(true);
-        ecsLayout.addTopLevelLabel("top_level");
         ecsLayout.setEventDataset("testdataset.log");
         ecsLayout.activateOptions();
         appender.setLayout(ecsLayout);

log4j2-ecs-layout/README.md

@@ -44,7 +44,6 @@ Instead of the usual `<PatternLayout/>`, use `<EcsLayout serviceName="my-app"/>`
 |-----------------|-------|-------|-----------|
 |serviceName      |String |       |Sets the `service.name` field so you can filter your logs by a particular service |
 |eventDataset     |String |`${serviceName}.log`|Sets the `event.dataset` field used by the machine learning job of the Logs app to look for anomalies in the log rate. |
-|topLevelLabels   |String |`trace.id, transaction.id, span.id, error.id, service.name`|Usually, MDC keys are nested under [`labels`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html). You can specify a comma-separated list of properties which should be on the top level. |
 |includeMarkers   |boolean|`false`|Log [Markers](https://logging.apache.org/log4j/2.0/manual/markers.html) as [`tags`](https://www.elastic.co/guide/en/ecs/current/ecs-base.html) |
 |stackTraceAsArray|boolean|`false`|Serializes the [`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html) as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex [Filebeat configuration](../README.md#when-stacktraceasarray-is-enabled).|
 |includeOrigin    |boolean|`false`|If `true`, adds the [`log.origin.file.name`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html), [`log.origin.file.line`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) and [`log.origin.function`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) fields. Note that you also have to set `includeLocation="true"` on your loggers and appenders if you are using the async ones. |

log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java

@@ -51,11 +51,7 @@
 import org.apache.logging.log4j.util.TriConsumer;
 
 import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -65,13 +61,10 @@ public class EcsLayout extends AbstractStringLayout {
     public static final Charset UTF_8 = Charset.forName("UTF-8");
     public static final String[] JSON_FORMAT = {"JSON"};
 
-    private final TriConsumer<String, Object, StringBuilder> WRITE_KEY_VALUES_INTO = new TriConsumer<String, Object, StringBuilder>() {
+    private final TriConsumer<String, Object, StringBuilder> WRITE_MDC = new TriConsumer<String, Object, StringBuilder>() {
         @Override
         public void accept(final String key, final Object value, final StringBuilder stringBuilder) {
             stringBuilder.append('\"');
-            if (!topLevelLabels.contains(key)) {
-                stringBuilder.append("labels.");
-            }
             JsonUtils.quoteAsString(key, stringBuilder);
             stringBuilder.append("\":\"");
             JsonUtils.quoteAsString(EcsJsonSerializer.toNullSafeString(String.valueOf(value)), stringBuilder);
@@ -81,7 +74,6 @@ public void accept(final String key, final Object value, final StringBuilder str
 
     private final KeyValuePair[] additionalFields;
     private final PatternFormatter[][] fieldValuePatternFormatter;
-    private final Set<String> topLevelLabels;
     private final boolean stackTraceAsArray;
     private final String serviceName;
     private final String eventDataset;
@@ -90,13 +82,11 @@ public void accept(final String key, final Object value, final StringBuilder str
     private final ConcurrentMap<Class<? extends MultiformatMessage>, Boolean> supportsJson = new ConcurrentHashMap<Class<? extends MultiformatMessage>, Boolean>();
     private final ObjectMessageJacksonSerializer objectMessageJacksonSerializer = ObjectMessageJacksonSerializer.Resolver.INSTANCE.resolve();
 
-    private EcsLayout(Configuration config, String serviceName, String eventDataset, boolean includeMarkers, KeyValuePair[] additionalFields, Collection<String> topLevelLabels, boolean includeOrigin, boolean stackTraceAsArray) {
+    private EcsLayout(Configuration config, String serviceName, String eventDataset, boolean includeMarkers, KeyValuePair[] additionalFields, boolean includeOrigin, boolean stackTraceAsArray) {
         super(config, UTF_8, null, null);
         this.serviceName = serviceName;
         this.eventDataset = eventDataset;
         this.includeMarkers = includeMarkers;
-        this.topLevelLabels = new HashSet<String>(topLevelLabels);
-        this.topLevelLabels.addAll(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS);
         this.includeOrigin = includeOrigin;
         this.stackTraceAsArray = stackTraceAsArray;
         this.additionalFields = additionalFields;
@@ -141,7 +131,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr
         EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
         EcsJsonSerializer.serializeThreadName(builder, event.getThreadName());
         EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName());
-        serializeLabels(event, builder);
+        serializeAdditionalFieldsAndMDC(event, builder);
         serializeTags(event, builder);
         if (includeOrigin) {
             EcsJsonSerializer.serializeOrigin(builder, event.getSource());
@@ -151,7 +141,7 @@ private StringBuilder toText(LogEvent event, StringBuilder builder, boolean gcFr
         return builder;
     }
 
-    private void serializeLabels(LogEvent event, StringBuilder builder) {
+    private void serializeAdditionalFieldsAndMDC(LogEvent event, StringBuilder builder) {
         final int length = additionalFields.length;
         if (!event.getContextData().isEmpty() || length > 0) {
             if (length > 0) {
@@ -185,7 +175,7 @@ private void serializeLabels(LogEvent event, StringBuilder builder) {
                     }
                 }
             }
-            event.getContextData().forEach(WRITE_KEY_VALUES_INTO, builder);
+            event.getContextData().forEach(WRITE_MDC, builder);
         }
     }
 
@@ -337,8 +327,6 @@ public static class Builder extends AbstractStringLayout.Builder<EcsLayout.Build
         private boolean stackTraceAsArray = false;
         @PluginElement("AdditionalField")
         private KeyValuePair[] additionalFields = new KeyValuePair[]{};
-        @PluginBuilderAttribute("topLevelLabels")
-        private String topLevelLabels;
         @PluginBuilderAttribute("includeOrigin")
         private boolean includeOrigin = false;
 
@@ -367,15 +355,6 @@ public boolean isIncludeOrigin() {
             return includeOrigin;
         }
 
-        public String getTopLevelLabels() {
-            return topLevelLabels;
-        }
-
-        public EcsLayout.Builder setTopLevelLabels(final String topLevelLabels) {
-            this.topLevelLabels = topLevelLabels;
-            return asBuilder();
-        }
-
         /**
          * Additional fields to set on each log event.
          *
@@ -413,13 +392,7 @@ public EcsLayout.Builder setStackTraceAsArray(boolean stackTraceAsArray) {
 
         @Override
         public EcsLayout build() {
-            List<String> topLevelLabelsList = new ArrayList<String>();
-            if (topLevelLabels != null) {
-                for (String label : topLevelLabels.split(",")) {
-                    topLevelLabelsList.add(label.trim());
-                }
-            }
-            return new EcsLayout(getConfiguration(), serviceName, EcsJsonSerializer.computeEventDataset(eventDataset, serviceName), includeMarkers, additionalFields, topLevelLabelsList, includeOrigin, stackTraceAsArray);
+            return new EcsLayout(getConfiguration(), serviceName, EcsJsonSerializer.computeEventDataset(eventDataset, serviceName), includeMarkers, additionalFields, includeOrigin, stackTraceAsArray);
         }
 
         public boolean isStackTraceAsArray() {

log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java

@@ -51,19 +51,17 @@ void tearDown() throws Exception {
     }
 
     @Test
-    void globalLabels() throws Exception {
+    void testAdditionalFields() throws Exception {
         putMdc("trace.id", "foo");
-        putMdc("top_level", "foo");
-        putMdc("nested_under_labels", "foo");
+        putMdc("foo", "bar");
         debug("test");
         assertThat(getLastLogLine().get("cluster.uuid").textValue()).isEqualTo("9fe9134b-20b0-465e-acf9-8cc09ac9053b");
         assertThat(getLastLogLine().get("node.id").textValue()).isEqualTo("foo");
         assertThat(getLastLogLine().get("empty")).isNull();
         assertThat(getLastLogLine().get("emptyPattern")).isNull();
         assertThat(getLastLogLine().get("clazz").textValue()).startsWith(getClass().getPackageName());
         assertThat(getLastLogLine().get("404")).isNull();
-        assertThat(getLastLogLine().get("top_level").textValue()).isEqualTo("foo");
-        assertThat(getLastLogLine().get("labels.nested_under_labels").textValue()).isEqualTo("foo");
+        assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
     }
 
     @Test

log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/Log4j2EcsLayoutTest.java

@@ -70,7 +70,6 @@ void setUp() {
                 .setIncludeMarkers(true)
                 .setIncludeOrigin(true)
                 .setStackTraceAsArray(true)
-                .setTopLevelLabels("top_level")
                 .setEventDataset("testdataset.log")
                 .setAdditionalFields(new KeyValuePair[]{
                         new KeyValuePair("cluster.uuid", "9fe9134b-20b0-465e-acf9-8cc09ac9053b"),

log4j2-ecs-layout/src/test/resources/log4j2-test.xml

@@ -5,7 +5,7 @@
     </Properties>
     <Appenders>
         <List name="TestAppender">
-            <EcsLayout serviceName="test" includeMarkers="true" includeOrigin="true" stackTraceAsArray="true" topLevelLabels="top_level"
+            <EcsLayout serviceName="test" includeMarkers="true" includeOrigin="true" stackTraceAsArray="true"
                        eventDataset="testdataset.log">
                 <KeyValuePair key="cluster.uuid" value="9fe9134b-20b0-465e-acf9-8cc09ac9053b"/>
                 <KeyValuePair key="node.id" value="${node.id}"/>

logback-ecs-encoder/src/main/java/co/elastic/logging/logback/EcsEncoder.java

@@ -35,10 +35,8 @@
 
 import java.nio.charset.Charset;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Set;
 
 public class EcsEncoder extends EncoderBase<ILoggingEvent> {
 
@@ -48,7 +46,6 @@ public class EcsEncoder extends EncoderBase<ILoggingEvent> {
     private String eventDataset;
     private boolean includeMarkers = false;
     private ThrowableProxyConverter throwableProxyConverter;
-    private Set<String> topLevelLabels = new HashSet<String>(EcsJsonSerializer.DEFAULT_TOP_LEVEL_LABELS);
     private boolean includeOrigin;
     private List<Pair> additionalFields = new ArrayList<Pair>();
 
@@ -77,7 +74,7 @@ public byte[] encode(ILoggingEvent event) {
         EcsJsonSerializer.serializeThreadName(builder, event.getThreadName());
         EcsJsonSerializer.serializeLoggerName(builder, event.getLoggerName());
         serializeAdditionalFields(builder);
-        EcsJsonSerializer.serializeLabels(builder, event.getMDCPropertyMap(), topLevelLabels);
+        EcsJsonSerializer.serializeMDC(builder, event.getMDCPropertyMap());
         if (includeOrigin) {
             StackTraceElement[] callerData = event.getCallerData();
             if (callerData != null && callerData.length > 0) {
@@ -110,10 +107,6 @@ private void serializeAdditionalFields(StringBuilder builder) {
         }
     }
 
-    public void addTopLevelLabel(String topLevelLabel) {
-        this.topLevelLabels.add(topLevelLabel);
-    }
-
     private void serializeMarkers(ILoggingEvent event, StringBuilder builder) {
         Marker marker = event.getMarker();
         if (includeMarkers && marker != null) {

logback-ecs-encoder/src/test/java/co/elastic/logging/logback/EcsEncoderTest.java

@@ -46,7 +46,6 @@ void setUp() {
         ecsEncoder.setIncludeMarkers(true);
         ecsEncoder.setStackTraceAsArray(true);
         ecsEncoder.setIncludeOrigin(true);
-        ecsEncoder.addTopLevelLabel("top_level");
         ecsEncoder.addAdditionalField(new EcsEncoder.Pair("foo", "bar"));
         ecsEncoder.addAdditionalField(new EcsEncoder.Pair("baz", "qux"));
         ecsEncoder.setEventDataset("testdataset.log");