feat: timestamp diff highlighting

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+- Add [`@timestamp` diff highlighting](README.md#timestamp-diff-highlighting):
+  the part of the timestamp that has changed from the preceding record is
+  underlined (in the default color scheme). This highlighting can be turned
+  off with the `timestampShowDiff: false` config var.
+  ([#20](https://github.com/trentm/go-ecslog/pull/20))
+
 - Add `ecsLenient: false` config option to allow rendering of lines that are
   likely ECS-compatible, but do not have all three required ecs-logging fields:
   `@timestamp`, `ecs.version`, `log.level`. Only one of those three is required

README.md

@@ -1,7 +1,8 @@
 # ecslog
 
-A CLI for pretty-printing (and filtering) of [ecs-logging](https://www.elastic.co/guide/en/ecs-logging/overview/master/intro.html)
-formatted log files.
+`ecslog` is a CLI for pretty-printing (and filtering) of log files in
+[ecs-logging](https://www.elastic.co/guide/en/ecs-logging/overview/master/intro.html)
+format.
 
 
 # Install
@@ -17,9 +18,23 @@ Or you can build from source via:
 
     git clone [email protected]:trentm/go-ecslog.git
     cd go-ecslog
-    make
+    make  # produces "./ecslog", a single binary you can put on your PATH
     ./ecslog --version
 
+# Features
+
+TODO: fill this out
+
+## `@timestamp` diff highlighting
+
+By default, `ecslog` will highlight the change in a log record's `@timestamp`
+from the previous log record. With the "default" formatter, the changed part
+of the timestamp is underlined. For example:
+
+![screenshot of @timestamp diff highlighting](./docs/img/timestamp-diff-highlighting.png)
+
+This can be turned off with the `timestampShowDiff=false` config var.
+
 
 # Goals
 
@@ -99,11 +114,11 @@ elide some fields, typically for compactness.
 
 # Config file
 
-Any of the following `ecslog` options can be set in a "~/.ecslog.toml" file.
+Any of the following `ecslog` options can be set in a `~/.ecslog.toml` file.
 See https://toml.io/ for TOML syntax information.  The `--no-config` option can
-be used to ignore "~/.ecslog.toml", if there is one.
+be used to ignore `~/.ecslog.toml`, if there is one.
 
-For example:
+An example config:
 
 ```
 format="compact"
@@ -117,7 +132,7 @@ Set the output format name (a string, equivalent of `-f, --format` option).
 Valid values are: "default" (the default), "compact", "ecs", "simple"
 
 ```
-format="compact"
+format="default"
 ```
 
 ### config: color
@@ -138,7 +153,7 @@ records.  Valid values are: -1 (to use the default 16384), or a value between 1
 and 1048576 (inclusive).
 
 ```
-maxLineLen=32768
+maxLineLen=16384
 ```
 
 ### config: ecsLenient
@@ -154,7 +169,17 @@ those three fields. Set `ecsLenient` to true to tell `ecslog` to attempt to
 rendering any log record that has **at least one** of these fields.
 
 ```
-ecsLenient=true
+ecsLenient=false
+```
+
+### config: timestampShowDiff
+
+If coloring the output (see [config: color](#config-color) above), by default
+`ecslog` will style the change in the timestamp from the preceding log record.
+Set this config var to `false` to turn off this styling.
+
+```
+timestampShowDiff=true
 ```
 
 

TODO.md

@@ -7,8 +7,6 @@
 - README needs a once-over
 - review TODOs in the code
 - clear out all panic()s and probably lo?g.Fatal()s? Perhaps remove from 'lg' pkg
-- releases: because unsigned dev thing on Mac, it would be nice to get into
-  brew. Look at goreleaser again
 
 # mvp
 
@@ -36,14 +34,11 @@
 # docs
 
 - specify that multifile behaviour may change later to merge on @timestamp
+  (https://github.com/trentm/go-ecslog/issues/16)
 
 # later
 
 - learn about verifiable builds: https://goreleaser.com/customization/gomod/
-- Is there a way to do releases for macOS and not have users hit the
-  "Developer cannot be verified" error?
-  https://stackoverflow.com/questions/59890359/developer-cannot-be-verified-macos-error-exception-a-move-to-trash-b-cancel
-  Tarball? Zip? Installer? Verifying with mac somehow (ew)? Brew tap?
 - Clickable file+linenum. If the ECS log record includes file name and line
   fields (https://www.elastic.co/guide/en/ecs/current/ecs-log.html#field-log-origin-file-line)
   the it would be nice if the default rendering of that (perhaps in the title line?)
@@ -54,6 +49,9 @@
       line number. What about ecslog (or another project) providing a small
       command to do that mapping (of log.origin.file.{name,line}) to local
       paths and/or GH links? Then provide docs on setting that up.
+- consider https://github.com/muesli/termenv for ANSI color handling.
+  Supports truecolor and degradation. Supports templates which might be
+  useful for config-based custom styling.
 - "http" output format -> fieldRenderers?
 - coloring for added zap and other levels (test case for this)
 - get ECS log examples from all the ecs-logging-$lang examples to learn from
@@ -77,6 +75,7 @@
 - benchmarking to be able to test out "TODO perf" ideas
 - godoc and examples (https://blog.golang.org/examples)
 
+
 # musing on custom formats/profiles
 
 ~/.ecslog.toml
@@ -109,7 +108,7 @@ a format now. *Or* could still be "format", but the default built-in formats
 
     ecslog -f trent
 
-If doing this (a format include the other attributes), then need to *not*
+If doing this (a format includes the other attributes), then need to *not*
 allow top-level attributes in config, i.e. NOT this
 
     # NOT this

cmd/ecslog/main.go

@@ -150,13 +150,19 @@ func main() {
 		ecsLenient = cfgECSLenient
 	}
 
+	timestampShowDiff := true
+	if cfgTimestampShowDiff, ok := cfg.GetBool("timestampShowDiff"); ok {
+		timestampShowDiff = cfgTimestampShowDiff
+	}
+
 	r, err := ecslog.NewRenderer(
 		shouldColorize,
 		*flagColorScheme,
 		formatName,
 		maxLineLen,
 		excludeFields,
 		ecsLenient,
+		timestampShowDiff,
 	)
 	if err != nil {
 		printError(err.Error())

examples/timestamp-diff-highlighting.log

@@ -0,0 +1,7 @@
+{"log.level":"debug","@timestamp":"2021-04-15T04:21:16.049Z","ecs":{"version":"1.5.0"},"message":"server started"}
+{"log.level":"debug","@timestamp":"2021-04-15T04:22:30.507Z","ecs":{"version":"1.5.0"},"message":"minutes later"}
+{"log.level":"debug","@timestamp":"2021-04-15T04:22:32.742Z","ecs":{"version":"1.5.0"},"message":"a few seconds later"}
+{"log.level":"debug","@timestamp":"2021-04-15T04:22:33.170Z","ecs":{"version":"1.5.0"},"message":"in the next second"}
+{"log.level":"debug","@timestamp":"2021-04-15T04:22:33.493Z","ecs":{"version":"1.5.0"},"message":"in the same second"}
+{"log.level":"debug","@timestamp":"2021-04-15T04:22:33.497Z","ecs":{"version":"1.5.0"},"message":"just a few ms later"}
+{"log.level":"debug","@timestamp":"2021-04-16T03:21:40.615Z","ecs":{"version":"1.5.0"},"message":"almost a day later"}

internal/ansipainter/ansipainter.go

@@ -100,7 +100,8 @@ type ANSIPainter struct {
 	painting    bool
 }
 
-// Paint ... TODO:doc
+// Paint with write the ANSI code to start styling with the ANSI SGR configured
+// for the given `role`.
 func (p *ANSIPainter) Paint(b *strings.Builder, role string) {
 	sgr, ok := p.sgrFromRole[role]
 	if ok {
@@ -111,11 +112,18 @@ func (p *ANSIPainter) Paint(b *strings.Builder, role string) {
 	}
 }
 
-// Reset ... TODO:doc
+// PaintAttrs will write the ANSI code to start styling with the given
+// attributes.
+func (p *ANSIPainter) PaintAttrs(b *strings.Builder, attrs []Attribute) {
+	b.WriteString(sgrFromAttrs(attrs))
+	p.painting = true
+}
+
+// Reset will write the ANSI SGR to reset styling, if necessary.
 func (p *ANSIPainter) Reset(b *strings.Builder) {
-	// TODO: skip this if there wasn't a code given to previous Paint -> p.painting
 	if p.painting {
 		b.WriteString(sgrReset)
+		p.painting = false
 	}
 }
 
@@ -125,19 +133,26 @@ func New(attrsFromRole map[string][]Attribute) *ANSIPainter {
 	p := ANSIPainter{}
 	p.sgrFromRole = make(map[string]string)
 	for role, attrs := range attrsFromRole {
-		sgr := escape + "["
-		for i, attr := range attrs {
-			if i > 0 {
-				sgr += ";"
-			}
-			sgr += strconv.Itoa(int(attr))
+		if len(attrs) > 0 {
+			p.sgrFromRole[role] = sgrFromAttrs(attrs)
 		}
-		sgr += "m"
-		p.sgrFromRole[role] = sgr
 	}
 	return &p
 }
 
+// sgrFromAttrs returns the ANSI escape code (SGR) for an array of attributes.
+func sgrFromAttrs(attrs []Attribute) string {
+	sgr := escape + "["
+	for i, attr := range attrs {
+		if i > 0 {
+			sgr += ";"
+		}
+		sgr += strconv.Itoa(int(attr))
+	}
+	sgr += "m"
+	return sgr
+}
+
 // NoColorPainter is a painter that emits no ANSI codes.
 var NoColorPainter = New(nil)
 
@@ -156,17 +171,23 @@ var BunyanPainter = New(map[string][]Attribute{
 var PinoPrettyPainter = New(map[string][]Attribute{
 	"message": {FgCyan},
 	"trace":   {FgHiBlack}, // FgHiBlack is chalk's conversion of "grey".
-	"debug":   {FgBlue},    // TODO: is this blue visible on cmd.exe?
+	"debug":   {FgBlue},    // TODO: is this blue visible on cmd.exe defaults?
 	"info":    {FgGreen},
 	"warn":    {FgYellow},
 	"error":   {FgRed},
 	"fatal":   {BgRed},
 })
 
 // DefaultPainter implements the stock default color scheme for `ecslog`.
-// TODO: test on windows
-// TODO: could add styles for punctuation (jq bolds them, I'd tend to make them faint)
+//
+// On timestamp styling: I wanted this to be somewhat subtle but not too subtle
+// to be able to read. I like timestampDiff=Underline or timestampSame=Faint,
+// but not both together. Anything else was too subtle (Italic) or too
+// distracting (fg or bg colors). Perhaps with True Color this could be better.
 var DefaultPainter = New(map[string][]Attribute{
+	"timestamp":     {},
+	"timestampSame": {},
+	"timestampDiff": {Underline},
 	"message":       {FgCyan},
 	"extraField":    {Bold},
 	"jsonObjectKey": {FgHiBlue},

internal/ecslog/ecslog.go

@@ -25,19 +25,22 @@ const defaultMaxLineLen = 16384
 
 // Renderer is the class used to drive ECS log rendering (aka pretty printing).
 type Renderer struct {
-	parser        fastjson.Parser
-	painter       *ansipainter.ANSIPainter
-	formatName    string
-	formatter     Formatter
-	maxLineLen    int
-	excludeFields []string
-	ecsLenient    bool
-	levelFilter   string
-	kqlFilter     *kqlog.Filter
-	strict        bool
-
-	line     []byte // the raw input line
-	logLevel string // cached "log.level", read during isECSLoggingRecord
+	parser            fastjson.Parser
+	painter           *ansipainter.ANSIPainter
+	formatName        string
+	formatter         Formatter
+	maxLineLen        int
+	excludeFields     []string
+	ecsLenient        bool
+	timestampShowDiff bool
+	levelFilter       string
+	kqlFilter         *kqlog.Filter
+	strict            bool
+
+	line             []byte // the raw input line
+	logLevel         string // cached "log.level", read during isECSLoggingRecord
+	lastTimestampBuf []byte // buffer to hold lastTimestamp values
+	lastTimestamp    []byte // last @timestamp (a slice of lastTimestampBuf)
 }
 
 // NewRenderer returns a new ECS logging log renderer.
@@ -54,7 +57,9 @@ type Renderer struct {
 //   to be an ecs-logging record it must have ecs.version, log.level, and
 //   @timestamp. If this option is true, it will only require *one* of those
 //   fields to exist.
-func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, excludeFields []string, ecsLenient bool) (*Renderer, error) {
+// - `timestampShowDiff` is a bool indicating if the @timestamp diff from the
+//   preceding log record should be styled.
+func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, excludeFields []string, ecsLenient, timestampShowDiff bool) (*Renderer, error) {
 	// Get appropriate "painter" for terminal coloring.
 	var painter *ansipainter.ANSIPainter
 	if shouldColorize == "auto" {
@@ -79,6 +84,8 @@ func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int,
 		}
 	case "no":
 		painter = ansipainter.NoColorPainter
+		// No point in calculating timestamp diff if not styling.
+		timestampShowDiff = false
 	default:
 		return nil, fmt.Errorf("invalid value for shouldColorize: %s", shouldColorize)
 	}
@@ -104,12 +111,17 @@ func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int,
 	lg.Printf("create renderer: formatName=%q, shouldColorize=%q, colorScheme=%q, maxLineLen=%d\n",
 		formatName, shouldColorize, colorScheme, maxLineLen)
 	return &Renderer{
-		painter:       painter,
-		formatName:    formatName,
-		formatter:     formatter,
-		maxLineLen:    maxLineLen,
-		excludeFields: excludeFields,
-		ecsLenient:    ecsLenient,
+		painter:           painter,
+		formatName:        formatName,
+		formatter:         formatter,
+		maxLineLen:        maxLineLen,
+		excludeFields:     excludeFields,
+		ecsLenient:        ecsLenient,
+		timestampShowDiff: timestampShowDiff,
+
+		// Can a timestamp ever reasonably be longer than 64 chars?
+		// "2021-04-15T04:22:29.507Z" is 24.
+		lastTimestampBuf: make([]byte, 64),
 	}, nil
 }