log: support maxBackups in config.toml

log/async_file_writer.go

@@ -3,13 +3,18 @@ package log
 import (
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 )
 
+const backupTimeFormat = "2006-01-02_15"
+
 type TimeTicker struct {
 	stop chan struct{}
 	C    <-chan time.Time
@@ -69,9 +74,11 @@ type AsyncFileWriter struct {
 	buf        chan []byte
 	stop       chan struct{}
 	timeTicker *TimeTicker
+
+	maxBackups int
 }
 
-func NewAsyncFileWriter(filePath string, maxBytesSize int64, rotateHours uint) *AsyncFileWriter {
+func NewAsyncFileWriter(filePath string, maxBytesSize int64, maxBackups int, rotateHours uint) *AsyncFileWriter {
 	absFilePath, err := filepath.Abs(filePath)
 	if err != nil {
 		panic(fmt.Sprintf("get file path of logger error. filePath=%s, err=%s", filePath, err))
@@ -81,6 +88,7 @@ func NewAsyncFileWriter(filePath string, maxBytesSize int64, rotateHours uint) *
 		filePath:   absFilePath,
 		buf:        make(chan []byte, maxBytesSize),
 		stop:       make(chan struct{}),
+		maxBackups: maxBackups,
 		timeTicker: NewTimeTicker(rotateHours),
 	}
 }
@@ -111,6 +119,8 @@ func (w *AsyncFileWriter) initLogFile() error {
 		return err
 	}
 
+	_ = w.clearBackups()
+
 	return nil
 }
 
@@ -222,5 +232,95 @@ func (w *AsyncFileWriter) flushAndClose() error {
 }
 
 func (w *AsyncFileWriter) timeFilePath(filePath string) string {
-	return filePath + "." + time.Now().Format("2006-01-02_15")
+	return filePath + "." + time.Now().Format(backupTimeFormat)
+}
+
+func (w *AsyncFileWriter) dir() string {
+	return filepath.Dir(w.filePath)
+}
+
+// oldLogFiles returns the list of backup log files stored in the same
+// directory as the current log file, sorted by ModTime
+func (w *AsyncFileWriter) oldLogFiles() ([]logInfo, error) {
+	files, err := os.ReadDir(w.dir())
+	if err != nil {
+		return nil, fmt.Errorf("can't read log file directory: %s", err)
+	}
+	logFiles := []logInfo{}
+
+	prefix := filepath.Base(w.filePath)
+
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+		k := f.Name()
+		if t, err := w.timeFromName(k, prefix); err == nil {
+			logFiles = append(logFiles, logInfo{t, f})
+		}
+	}
+
+	sort.Sort(byFormatTime(logFiles))
+
+	return logFiles, nil
+}
+
+// logInfo is a convenience struct to return the filename and its embedded
+// timestamp.
+type logInfo struct {
+	timestamp time.Time
+	fs.DirEntry
+}
+
+// byFormatTime sorts by newest time formatted in the name.
+type byFormatTime []logInfo
+
+func (b byFormatTime) Less(i, j int) bool {
+	return b[i].timestamp.After(b[j].timestamp)
+}
+
+func (b byFormatTime) Swap(i, j int) {
+	b[i], b[j] = b[j], b[i]
+}
+
+func (b byFormatTime) Len() int {
+	return len(b)
+}
+
+func (w *AsyncFileWriter) timeFromName(filename, prefix string) (time.Time, error) {
+	if !strings.HasPrefix(filename, prefix) || len(filename) == len(prefix) {
+		return time.Time{}, errors.New("mismatched prefix")
+	}
+	ts := filename[len(prefix)+1:]
+	a, _ := time.Parse(backupTimeFormat, ts)
+	return a, nil
+}
+
+func (w *AsyncFileWriter) clearBackups() error {
+	if w.maxBackups == 0 {
+		return nil
+	}
+	files, err := w.oldLogFiles()
+	if err != nil {
+		return err
+	}
+	var remove []logInfo
+	if w.maxBackups > 0 && w.maxBackups < len(files) {
+		preserved := make(map[string]bool)
+		for _, f := range files {
+			fn := f.Name()
+			preserved[fn] = true
+
+			if len(preserved) > w.maxBackups {
+				remove = append(remove, f)
+			}
+		}
+	}
+	for _, f := range remove {
+		errRemove := os.Remove(filepath.Join(w.dir(), f.Name()))
+		if err == nil && errRemove != nil {
+			err = errRemove
+		}
+	}
+	return err
 }

log/async_file_writer_test.go

@@ -2,14 +2,17 @@ package log
 
 import (
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestWriterHourly(t *testing.T) {
-	w := NewAsyncFileWriter("./hello.log", 100, 1)
+	w := NewAsyncFileWriter("./hello.log", 100, 1, 1)
 	w.Start()
 	w.Write([]byte("hello\n"))
 	w.Write([]byte("world\n"))
@@ -67,3 +70,24 @@ func TestGetNextRotationHour(t *testing.T) {
 		t.Run("TestGetNextRotationHour_"+strconv.Itoa(i), test(tc.now, tc.delta, tc.expectedHour))
 	}
 }
+
+func TestClearBackups(t *testing.T) {
+	dir := "./test"
+	os.Mkdir(dir, 0700)
+	w := NewAsyncFileWriter("./test/bsc.log", 100, 1, 1)
+	defer os.RemoveAll(dir)
+	fakeCurrentTime := time.Now()
+	name := ""
+	data := []byte("data")
+	for i := 0; i < 5; i++ {
+		name = w.filePath + "." + fakeCurrentTime.Format(backupTimeFormat)
+		_ = os.WriteFile(name, data, 0700)
+		fakeCurrentTime = fakeCurrentTime.Add(time.Hour * 1)
+	}
+	oldFiles, _ := w.oldLogFiles()
+	assert.True(t, len(oldFiles) == 5)
+	w.clearBackups()
+	remainFiles, _ := w.oldLogFiles()
+	assert.True(t, len(remainFiles) == 1)
+	assert.Equal(t, remainFiles[0].Name(), filepath.Base(name))
+}

log/handler.go

@@ -75,14 +75,14 @@ func FileHandler(path string, fmtr Format) (Handler, error) {
 // RotatingFileHandler returns a handler which writes log records to file chunks
 // at the given path. When a file's size reaches the limit, the handler creates
 // a new file named after the timestamp of the first log record it will contain.
-func RotatingFileHandler(filePath string, limit uint, formatter Format, rotateHours uint) (Handler, error) {
+func RotatingFileHandler(filePath string, limit uint, maxBackups uint, formatter Format, rotateHours uint) (Handler, error) {
 	if _, err := os.Stat(path.Dir(filePath)); os.IsNotExist(err) {
 		err := os.MkdirAll(path.Dir(filePath), 0755)
 		if err != nil {
 			return nil, fmt.Errorf("could not create directory %s, %v", path.Dir(filePath), err)
 		}
 	}
-	fileWriter := NewAsyncFileWriter(filePath, int64(limit), rotateHours)
+	fileWriter := NewAsyncFileWriter(filePath, int64(limit), int(maxBackups), rotateHours)
 	fileWriter.Start()
 	return StreamHandler(fileWriter, formatter), nil
 }

log/logger.go

@@ -290,8 +290,8 @@ func (c Ctx) toArray() []interface{} {
 	return arr
 }
 
-func NewFileLvlHandler(logPath string, maxBytesSize uint, level string, rotateHours uint) Handler {
-	rfh, err := RotatingFileHandler(logPath, maxBytesSize, LogfmtFormat(), rotateHours)
+func NewFileLvlHandler(logPath string, maxBytesSize uint, maxBackups uint, level string, rotateHours uint) Handler {
+	rfh, err := RotatingFileHandler(logPath, maxBytesSize, maxBackups, LogfmtFormat(), rotateHours)
 	if err != nil {
 		panic(err)
 	}

node/config.go

@@ -513,6 +513,7 @@ type LogConfig struct {
 	MaxBytesSize *uint   `toml:",omitempty"`
 	Level        *string `toml:",omitempty"`
 	RotateHours  *uint   `toml:",omitempty"`
+	MaxBackups   *uint   `toml:",omitempty"`
 
 	// TermTimeFormat is the time format used for console logging.
 	TermTimeFormat *string `toml:",omitempty"`

node/node.go

@@ -118,7 +118,12 @@ func New(conf *Config) (*Node, error) {
 				rotateHours = *conf.LogConfig.RotateHours
 			}
 
-			log.Root().SetHandler(log.NewFileLvlHandler(logFilePath, *conf.LogConfig.MaxBytesSize, *conf.LogConfig.Level, rotateHours))
+			maxBackups := uint(0)
+			if conf.LogConfig.MaxBackups != nil {
+				maxBackups = *conf.LogConfig.MaxBackups
+			}
+
+			log.Root().SetHandler(log.NewFileLvlHandler(logFilePath, *conf.LogConfig.MaxBytesSize, maxBackups, *conf.LogConfig.Level, rotateHours))
 		}
 	}
 	if conf.Logger == nil {