Add JSON log parser (#179)

* Add JSON log parser * Update README.adoc Fixed some grammatical errors. Co-authored-by: Martin Helmich <[email protected]> * Update parser/jsonparser/jsonparser_test.go Use the assertions of testify package. Co-authored-by: Martin Helmich <[email protected]> * Update parser/jsonparser/jsonparser.go Optimize performance. Co-authored-by: Martin Helmich <[email protected]> * Update jsonparser_test.go Use the testify package. * 1. Add Parser flag in LoadConfigFromFlags function. 2. Use testify package in parser/*._test file. Co-authored-by: Martin Helmich <[email protected]>
martin-helmich · Feb 25, 2021 · b033474 · b033474
1 parent dabff82
commit b033474
Show file tree

Hide file tree

Showing 11 changed files with 192 additions and 11 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.11
+FROM golang:1.13
 
 COPY . /work
 WORKDIR /work

diff --git a/README.adoc b/README.adoc
@@ -533,6 +533,10 @@ relabel "request_uri" {
 }
 ----
 
+### JSON log_format
+
+You can use the JSON parser by setting the `--parser` command line flag or `parser` config file property to `json`.
+
 == Frequently Asked Questions
 
 > I have started the exporter, but it is not exporting any application-specific metrics!

diff --git a/config/loader_flags.go b/config/loader_flags.go
@@ -11,6 +11,7 @@ func LoadConfigFromFlags(config *Config, flags *StartupFlags) error {
 	config.Namespaces = []NamespaceConfig{
 		{
 			Format: flags.Format,
+			Parser: flags.Parser,
 			Name:   flags.Namespace,
 			SourceData: SourceData{
 				Files: flags.Filenames,

diff --git a/config/struct_namespace.go b/config/struct_namespace.go
@@ -19,8 +19,9 @@ type NamespaceConfig struct {
 
 	SourceFiles      []string          `hcl:"source_files" yaml:"source_files"`
 	SourceData       SourceData        `hcl:"source" yaml:"source"`
-	Format           string            `hcl:"format"`
-	Labels           map[string]string `hcl:"labels"`
+	Parser           string            `hcl:"parser" yaml:"parser"`
+	Format           string            `hcl:"format" yaml:"format"`
+	Labels           map[string]string `hcl:"labels" yaml:"labels"`
 	RelabelConfigs   []RelabelConfig   `hcl:"relabel" yaml:"relabel_configs"`
 	HistogramBuckets []float64         `hcl:"histogram_buckets" yaml:"histogram_buckets"`
 

diff --git a/config/structs.go b/config/structs.go
@@ -5,6 +5,7 @@ package config
 type StartupFlags struct {
 	ConfigFile                 string
 	Filenames                  []string
+	Parser                     string
 	Format                     string
 	Namespace                  string
 	ListenPort                 int
@@ -78,4 +79,4 @@ func (l *ListenConfig) MetricsEndpointOrDefault() string {
 	}
 
 	return l.MetricsEndpoint
-}
+}
diff --git a/main.go b/main.go
@@ -30,12 +30,12 @@ import (
 
 	"github.com/martin-helmich/prometheus-nginxlog-exporter/config"
 	"github.com/martin-helmich/prometheus-nginxlog-exporter/discovery"
+	"github.com/martin-helmich/prometheus-nginxlog-exporter/parser"
 	"github.com/martin-helmich/prometheus-nginxlog-exporter/prof"
 	"github.com/martin-helmich/prometheus-nginxlog-exporter/relabeling"
 	"github.com/martin-helmich/prometheus-nginxlog-exporter/tail"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/satyrius/gonx"
 )
 
 type NSMetrics struct {
@@ -180,6 +180,7 @@ func main() {
 	nsGatherers := make(prometheus.Gatherers, 0)
 
 	flag.IntVar(&opts.ListenPort, "listen-port", 4040, "HTTP port to listen on")
+	flag.StringVar(&opts.Parser, "parser", "text", "NGINX access log format parser. One of: [text, json]")
 	flag.StringVar(&opts.Format, "format", `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"`, "NGINX access log format")
 	flag.StringVar(&opts.Namespace, "namespace", "nginx", "namespace to use for metric names")
 	flag.StringVar(&opts.ConfigFile, "config-file", "", "Configuration file to read from")
@@ -295,7 +296,7 @@ func setupConsul(cfg *config.Config, stopChan <-chan bool, stopHandlers *sync.Wa
 func processNamespace(nsCfg config.NamespaceConfig, metrics *Metrics) {
 	var followers []tail.Follower
 
-	parser := gonx.NewParser(nsCfg.Format)
+	parser := parser.NewParser(nsCfg)
 
 	for _, f := range nsCfg.SourceData.Files {
 		t, err := tail.NewFileFollower(f)
@@ -348,7 +349,7 @@ func processNamespace(nsCfg config.NamespaceConfig, metrics *Metrics) {
 
 }
 
-func processSource(nsCfg config.NamespaceConfig, t tail.Follower, parser *gonx.Parser, metrics *Metrics, hasCounterOnlyLabels bool) {
+func processSource(nsCfg config.NamespaceConfig, t tail.Follower, parser parser.Parser, metrics *Metrics, hasCounterOnlyLabels bool) {
 	relabelings := relabeling.NewRelabelings(nsCfg.RelabelConfigs)
 	relabelings = append(relabelings, relabeling.DefaultRelabelings...)
 	relabelings = relabeling.UniqueRelabelings(relabelings)
@@ -368,15 +369,13 @@ func processSource(nsCfg config.NamespaceConfig, t tail.Follower, parser *gonx.P
 			fmt.Println(line)
 		}
 
-		entry, err := parser.ParseString(line)
+		fields, err := parser.ParseString(line)
 		if err != nil {
 			fmt.Printf("error while parsing line '%s': %s\n", line, err)
 			metrics.parseErrorsTotal.Inc()
 			continue
 		}
 
-		fields := entry.Fields()
-
 		for i := range relabelings {
 			if str, ok := fields[relabelings[i].SourceValue]; ok {
 				mapped, err := relabelings[i].Map(str)
@@ -415,7 +414,7 @@ func processSource(nsCfg config.NamespaceConfig, t tail.Follower, parser *gonx.P
 	}
 }
 
-func floatFromFields(fields gonx.Fields, name string) (float64, bool) {
+func floatFromFields(fields map[string]string, name string) (float64, bool) {
 	val, ok := fields[name]
 	if !ok {
 		return 0, false

diff --git a/parser/jsonparser/jsonparser.go b/parser/jsonparser/jsonparser.go
@@ -0,0 +1,34 @@
+package jsonparser
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// JsonParser parse a JSON string.
+type JsonParser struct{}
+
+// NewJsonParser returns a new json parser.
+func NewJsonParser() *JsonParser {
+	return &JsonParser{}
+}
+
+// ParseString implements the Parser interface.
+// The value in the map is not necessarily a string, so it needs to be converted.
+func (j *JsonParser) ParseString(line string) (map[string]string, error) {
+	var parsed map[string]interface{}
+	err := json.Unmarshal([]byte(line), &parsed)
+	if err != nil {
+		return nil, fmt.Errorf("json log parsing err: %w", err)
+	}
+
+	fields := make(map[string]string, len(parsed))
+	for k, v := range parsed {
+		if s, ok := v.(string); ok {
+			fields[k] = s
+		} else {
+			fields[k] = fmt.Sprintf("%v", v)
+		}
+	}
+	return fields, nil
+}
diff --git a/parser/jsonparser/jsonparser_test.go b/parser/jsonparser/jsonparser_test.go
@@ -0,0 +1,44 @@
+package jsonparser
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestJsonParse(t *testing.T) {
+	parser := NewJsonParser()
+	line := `{"time_local":"2021-02-03T11:22:33+08:00","request_length":123,"request_method":"GET","request":"GET /order/2145 HTTP/1.1","body_bytes_sent":518,"status": 200,"request_time":0.544,"upstream_response_time":"0.543"}`
+
+	got, err := parser.ParseString(line)
+	require.NoError(t, err)
+
+	want := map[string]string{
+		"time_local":             "2021-02-03T11:22:33+08:00",
+		"request_time":           "0.544",
+		"request_length":         "123",
+		"upstream_response_time": "0.543",
+		"status":                 "200",
+		"body_bytes_sent":        "518",
+		"request":                "GET /order/2145 HTTP/1.1",
+		"request_method":         "GET",
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("JsonParser.Parse() = %v, want %v", got, want)
+	}
+}
+
+func BenchmarkParseJson(b *testing.B) {
+	parser := NewJsonParser()
+	line := `{"time_local":"2021-02-03T11:22:33+08:00","request_length":123,"request_method":"GET","request":"GET /order/2145 HTTP/1.1","body_bytes_sent":518,"status": 200,"request_time":0.544,"upstream_response_time":"0.543"}`
+
+	for i := 0; i < b.N; i++ {
+		res, err := parser.ParseString(line)
+		if err != nil {
+			b.Error(err)
+		}
+		_ = fmt.Sprintf("%v", res)
+	}
+}
diff --git a/parser/parser.go b/parser/parser.go
@@ -0,0 +1,24 @@
+package parser
+
+import (
+	"github.com/martin-helmich/prometheus-nginxlog-exporter/config"
+	"github.com/martin-helmich/prometheus-nginxlog-exporter/parser/jsonparser"
+	"github.com/martin-helmich/prometheus-nginxlog-exporter/parser/textparser"
+)
+
+// Parser parses a line of log to a map[string]string.
+type Parser interface {
+	ParseString(line string) (map[string]string, error)
+}
+
+// NewParser returns a Parser with the given config.NamespaceConfig.
+func NewParser(nsCfg config.NamespaceConfig) Parser {
+	switch nsCfg.Parser {
+	case "text":
+		return textparser.NewTextParser(nsCfg.Format)
+	case "json":
+		return jsonparser.NewJsonParser()
+	default:
+		return textparser.NewTextParser(nsCfg.Format)
+	}
+}
diff --git a/parser/textparser/textparser.go b/parser/textparser/textparser.go
@@ -0,0 +1,29 @@
+package textparser
+
+import (
+	"fmt"
+
+	"github.com/satyrius/gonx"
+)
+
+// TextParser parses variables patterns using config.NamespaceConfig.Format.
+type TextParser struct {
+	parser *gonx.Parser
+}
+
+// NewTextParser returns a new text parser.
+func NewTextParser(format string) *TextParser {
+	return &TextParser{
+		parser: gonx.NewParser(format),
+	}
+}
+
+// ParseString implements the Parser interface.
+func (t *TextParser) ParseString(line string) (map[string]string, error) {
+	entry, err := t.parser.ParseString(line)
+	if err != nil {
+		return nil, fmt.Errorf("text log parsing err: %w", err)
+	}
+
+	return entry.Fields(), nil
+}
diff --git a/parser/textparser/textparser_test.go b/parser/textparser/textparser_test.go
@@ -0,0 +1,44 @@
+package textparser
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTextParse(t *testing.T) {
+	parser := NewTextParser(`[$time_local] $request_method "$request" $request_length $body_bytes_sent $status $request_time $upstream_response_time`)
+	line := `[03/Feb/2021:11:22:33 +0800] GET "GET /order/2145 HTTP/1.1" 123 518 200 0.544 0.543`
+
+	got, err := parser.ParseString(line)
+	require.NoError(t, err)
+
+	want := map[string]string{
+		"time_local":             "03/Feb/2021:11:22:33 +0800",
+		"request_time":           "0.544",
+		"request_length":         "123",
+		"upstream_response_time": "0.543",
+		"status":                 "200",
+		"body_bytes_sent":        "518",
+		"request":                "GET /order/2145 HTTP/1.1",
+		"request_method":         "GET",
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("TextParser.Parse() = %v, want %v", got, want)
+	}
+}
+
+func BenchmarkParseText(b *testing.B) {
+	parser := NewTextParser(`[$time_local] $request_method "$request" $request_length $body_bytes_sent $status $request_time $upstream_response_time`)
+	line := `[03/Feb/2021:11:22:33 +0800] GET "GET /order/2145 HTTP/1.1" 123 518 200 0.544 0.543`
+
+	for i := 0; i < b.N; i++ {
+		res, err := parser.ParseString(line)
+		if err != nil {
+			b.Error(err)
+		}
+		_ = fmt.Sprintf("%v", res)
+	}
+}