From 78d4a3b7743cb5d6ee5242674e303b491f0314d6 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Wed, 21 Feb 2024 17:32:31 +0800 Subject: [PATCH 1/7] suport compress --- config.go | 4 +++- go.mod | 3 +-- go.sum | 6 ++---- log.go | 6 ++++++ 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/config.go b/config.go index 2d581d1..30272e6 100644 --- a/config.go +++ b/config.go @@ -35,6 +35,8 @@ type FileLogConfig struct { MaxDays int `toml:"max-days" json:"max-days"` // Maximum number of old log files to retain. MaxBackups int `toml:"max-backups" json:"max-backups"` + // Compress function for rotated files. + Compress string `toml:"compress" json:"compress"` } // Config serializes log related config in toml/json. @@ -103,7 +105,7 @@ func (cfg *Config) buildOptions(errSink zapcore.WriteSyncer) []zap.Option { if cfg.Sampling != nil { opts = append(opts, zap.WrapCore(func(core zapcore.Core) zapcore.Core { - return zapcore.NewSampler(core, time.Second, int(cfg.Sampling.Initial), int(cfg.Sampling.Thereafter)) + return zapcore.NewSamplerWithOptions(core, time.Second, int(cfg.Sampling.Initial), int(cfg.Sampling.Thereafter)) })) } return opts diff --git a/go.mod b/go.mod index b72bdcb..b588aeb 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,12 @@ module github.com/pingcap/log require ( - github.com/BurntSushi/toml v0.3.1 // indirect github.com/pingcap/errors v0.11.0 github.com/stretchr/testify v1.7.0 go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.19.0 - gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) go 1.16 diff --git a/go.sum b/go.sum index 82e0db7..b60eedd 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -46,8 +44,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/log.go b/log.go index 01edeb8..f872e01 100644 --- a/log.go +++ b/log.go @@ -182,6 +182,11 @@ func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) { cfg.MaxSize = defaultLogMaxSize } + compress := false + if cfg.Compress == "gzip" { + compress = true + } + // use lumberjack to logrotate return &lumberjack.Logger{ Filename: cfg.Filename, @@ -189,6 +194,7 @@ func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) { MaxBackups: cfg.MaxBackups, MaxAge: cfg.MaxDays, LocalTime: true, + Compress: compress, }, nil } From 182a2d5687ef0094323c268d05b4979f753139ad Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Thu, 22 Feb 2024 14:44:11 +0800 Subject: [PATCH 2/7] update test --- zap_log_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/zap_log_test.go b/zap_log_test.go index 4075297..1ac359d 100644 --- a/zap_log_test.go +++ b/zap_log_test.go @@ -235,6 +235,44 @@ func TestLogJSON(t *testing.T) { "{\"level\":\"INFO\",\"caller\":\"zap_log_test.go:233\",\"message\":\"new connection\",\"connID\":\"1\",\"traceID\":\"dse1121\"}") } +func TestRotateLogWithCompress(t *testing.T) { + tempDir := t.TempDir() + conf := &Config{ + Level: "info", + File: FileLogConfig{ + Filename: tempDir + "/test.log", + MaxSize: 1, + Compress: "gzip", + }, + } + logger, _, err := InitLogger(conf) + require.NoError(t, err) + + var data []byte + for i := 1; i <= 1*1024*1024; i++ { + if i%1000 != 0 { + data = append(data, 'd') + continue + } + logger.Info(string(data)) + data = data[:0] + } + // Waiting rotation finished, it's async + for { + files, _ := os.ReadDir(tempDir) + if len(files) == 2 { + for _, f := range files { + info, err := f.Info() + require.NoError(t, err) + // Many duplicate logs, the file size after compress should less than 512KB + require.Less(t, info.Size(), int64(512*1024)) + } + break + } + time.Sleep(time.Millisecond) + } +} + // testLogSpy is a testing.TB that captures logged messages. type testLogSpy struct { testing.TB From 0e523e83f6789414ef60fb12a658d7e404b76642 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Thu, 22 Feb 2024 15:01:28 +0800 Subject: [PATCH 3/7] update comments --- config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/config.go b/config.go index 30272e6..75e0f9d 100644 --- a/config.go +++ b/config.go @@ -36,6 +36,7 @@ type FileLogConfig struct { // Maximum number of old log files to retain. MaxBackups int `toml:"max-backups" json:"max-backups"` // Compress function for rotated files. + // Currently only `gzip` is supported, otherwise compression will be disabled. Compress string `toml:"compress" json:"compress"` } From a6efc02bb48413b33b567a45592345aa0e250193 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Thu, 22 Feb 2024 15:32:45 +0800 Subject: [PATCH 4/7] update --- log.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/log.go b/log.go index f872e01..8713b40 100644 --- a/log.go +++ b/log.go @@ -183,8 +183,13 @@ func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) { } compress := false - if cfg.Compress == "gzip" { + switch cfg.Compress { + case "": + compress = false + case "gzip": compress = true + default: + return nil, fmt.Errorf("can't set compress to `%s`", cfg.Compress) } // use lumberjack to logrotate From 1e467ed3d218d630382f3bce5d8a11dc9be51c64 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Thu, 22 Feb 2024 15:33:38 +0800 Subject: [PATCH 5/7] update test --- zap_log_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zap_log_test.go b/zap_log_test.go index 1ac359d..153141d 100644 --- a/zap_log_test.go +++ b/zap_log_test.go @@ -273,6 +273,20 @@ func TestRotateLogWithCompress(t *testing.T) { } } +func TestCompressError(t *testing.T) { + tempDir := t.TempDir() + conf := &Config{ + Level: "info", + File: FileLogConfig{ + Filename: tempDir + "/test.log", + MaxSize: 1, + Compress: "xxx", + }, + } + _, _, err := InitLogger(conf) + require.Error(t, err) +} + // testLogSpy is a testing.TB that captures logged messages. type testLogSpy struct { testing.TB From 42379a095d1bd194e6d7fca624a2f0201c071f96 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Thu, 22 Feb 2024 15:39:24 +0800 Subject: [PATCH 6/7] update --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 75e0f9d..c64e11b 100644 --- a/config.go +++ b/config.go @@ -36,7 +36,7 @@ type FileLogConfig struct { // Maximum number of old log files to retain. MaxBackups int `toml:"max-backups" json:"max-backups"` // Compress function for rotated files. - // Currently only `gzip` is supported, otherwise compression will be disabled. + // Currently only `gzip` and empty are supported, empty means disables compression. Compress string `toml:"compress" json:"compress"` } From a62f0360f01260326292bccc9f16ee53b817ea1d Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Thu, 22 Feb 2024 15:40:15 +0800 Subject: [PATCH 7/7] update --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index c64e11b..8ce54aa 100644 --- a/config.go +++ b/config.go @@ -36,7 +36,7 @@ type FileLogConfig struct { // Maximum number of old log files to retain. MaxBackups int `toml:"max-backups" json:"max-backups"` // Compress function for rotated files. - // Currently only `gzip` and empty are supported, empty means disables compression. + // Currently only `gzip` and empty are supported, empty means compression disabled. Compress string `toml:"compress" json:"compress"` }