log: Support a dedicated log file to save general log

DEPS.bzl

@@ -5871,26 +5871,26 @@ def go_deps():
         name = "com_github_pingcap_log",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/log",
-        sha256 = "9b0ae182fcc611cc535eb18a6f332846f0744b905e48888c06c1f6aeda8036d5",
-        strip_prefix = "github.com/pingcap/[email protected].20230317032135-a0d097d16e22",
+        sha256 = "c393f943bb688e240dcf9a50ac1d915aa27371803524887ab61a235834bb7438",
+        strip_prefix = "github.com/pingcap/[email protected].20240314023424-862ccc32f18d",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20230317032135-a0d097d16e22.zip",
-            "http://ats.apps.svc/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20230317032135-a0d097d16e22.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20230317032135-a0d097d16e22.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20230317032135-a0d097d16e22.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20240314023424-862ccc32f18d.zip",
+            "http://ats.apps.svc/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20240314023424-862ccc32f18d.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20240314023424-862ccc32f18d.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/log/com_github_pingcap_log-v1.1.1-0.20240314023424-862ccc32f18d.zip",
         ],
     )
     go_repository(
         name = "com_github_pingcap_sysutil",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/sysutil",
-        sha256 = "0540f3dfd25706302c0f484baae0b6bb6a6382893bbf8b14720debcb9756f1d0",
-        strip_prefix = "github.com/pingcap/[email protected].20230407040306-fb007c5aff21",
+        sha256 = "c2ffa4a6d163ca73e73831de5abaa25d47abba59c41b8a549d70935442921a56",
+        strip_prefix = "github.com/pingcap/[email protected].20240311050922-ae81ee01f3a5",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20230407040306-fb007c5aff21.zip",
-            "http://ats.apps.svc/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20230407040306-fb007c5aff21.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20230407040306-fb007c5aff21.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20230407040306-fb007c5aff21.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20240311050922-ae81ee01f3a5.zip",
+            "http://ats.apps.svc/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20240311050922-ae81ee01f3a5.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20240311050922-ae81ee01f3a5.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/sysutil/com_github_pingcap_sysutil-v1.0.1-0.20240311050922-ae81ee01f3a5.zip",
         ],
     )
     go_repository(

cmd/benchdb/main.go

@@ -56,7 +56,7 @@ var (
 func main() {
 	flag.Parse()
 	flag.PrintDefaults()
-	err := logutil.InitLogger(logutil.NewLogConfig(*logLevel, logutil.DefaultLogFormat, "", logutil.EmptyFileLogConfig, false))
+	err := logutil.InitLogger(logutil.NewLogConfig(*logLevel, logutil.DefaultLogFormat, "", "", logutil.EmptyFileLogConfig, false))
 	terror.MustNil(err)
 	err = store.Register("tikv", driver.TiKVDriver{})
 	terror.MustNil(err)

cmd/tidb-server/main.go

@@ -108,6 +108,7 @@ const (
 	nmLogLevel         = "L"
 	nmLogFile          = "log-file"
 	nmLogSlowQuery     = "log-slow-query"
+	nmLogGeneral       = "log-general"
 	nmReportStatus     = "report-status"
 	nmStatusHost       = "status-host"
 	nmStatusPort       = "status"
@@ -163,6 +164,7 @@ var (
 	logLevel     *string
 	logFile      *string
 	logSlowQuery *string
+	logGeneral   *string
 
 	// Status
 	reportStatus    *bool
@@ -216,6 +218,7 @@ func initFlagSet() *flag.FlagSet {
 	logLevel = fset.String(nmLogLevel, "info", "log level: info, debug, warn, error, fatal")
 	logFile = fset.String(nmLogFile, "", "log file path")
 	logSlowQuery = fset.String(nmLogSlowQuery, "", "slow query file path")
+	logGeneral = fset.String(nmLogGeneral, "", "general log file path")
 
 	// Status
 	reportStatus = flagBoolean(fset, nmReportStatus, true, "If enable status report HTTP service.")
@@ -580,6 +583,9 @@ func overrideConfig(cfg *config.Config, fset *flag.FlagSet) {
 	if actualFlags[nmLogSlowQuery] {
 		cfg.Log.SlowQueryFile = *logSlowQuery
 	}
+	if actualFlags[nmLogGeneral] {
+		cfg.Log.GeneralLogFile = *logGeneral
+	}
 
 	// Status
 	if actualFlags[nmReportStatus] {

go.mod

@@ -85,8 +85,8 @@ require (
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c
 	github.com/pingcap/fn v1.0.0
 	github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754
-	github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22
-	github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21
+	github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d
+	github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5
 	github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e
 	github.com/pingcap/tipb v0.0.0-20240116032918-9bb28c43bbfc
 	github.com/prometheus/client_golang v1.19.0

go.sum

@@ -719,10 +719,10 @@ github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 h1:nU9wDeMsID8EWaw
 github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
 github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
 github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
-github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 h1:2SOzvGvE8beiC1Y4g9Onkvu6UmuBBOeWRGQEjJaT/JY=
-github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
-github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I=
-github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM=
+github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfUnQGqft0ud+xVFuCdp1XkVL0X1E=
+github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA=
+github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE=
+github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530=
 github.com/pingcap/tipb v0.0.0-20240116032918-9bb28c43bbfc h1:sEp4lbExDfnMX8HXQyhZrhqo2/SgeFY5KOdo5akc8FM=
 github.com/pingcap/tipb v0.0.0-20240116032918-9bb28c43bbfc/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
@@ -1474,6 +1474,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/config/config.go

@@ -490,6 +490,8 @@ type Log struct {
 	// ExpensiveThreshold is deprecated.
 	ExpensiveThreshold uint `toml:"expensive-threshold" json:"expensive-threshold"`
 
+	GeneralLogFile string `toml:"general-log-file" json:"general-log-file"`
+
 	// The following items are deprecated. We need to keep them here temporarily
 	// to support the upgrade process. They can be removed in future.
 
@@ -1441,7 +1443,7 @@ var TableLockDelayClean = func() uint64 {
 
 // ToLogConfig converts *Log to *logutil.LogConfig.
 func (l *Log) ToLogConfig() *logutil.LogConfig {
-	return logutil.NewLogConfig(l.Level, l.Format, l.SlowQueryFile, l.File, l.getDisableTimestamp(),
+	return logutil.NewLogConfig(l.Level, l.Format, l.SlowQueryFile, l.GeneralLogFile, l.File, l.getDisableTimestamp(),
 		func(config *zaplog.Config) { config.DisableErrorVerbose = l.getDisableErrorStack() },
 		func(config *zaplog.Config) { config.Timeout = l.Timeout },
 	)

pkg/config/config.toml.example

@@ -154,6 +154,10 @@ format = "text"
 # Stores slow query log into separated files.
 slow-query-file = "tidb-slow.log"
 
+# Stores general log into separated files.
+# If it equals to empty, the general log will be written to the server log.
+general-log-file = ""
+
 # Make tidb panic if the write log operation hang for 15s
 # timeout = 15
 
@@ -171,6 +175,10 @@ max-days = 0
 # Maximum number of old log files to retain. No clean up by default.
 max-backups = 0
 
+# Compression function for rotated files.
+# It can be empty or "gzip", empty means compression disabled.
+compression = ""
+
 [security]
 # Path of file that contains list of trusted SSL CAs for connection with mysql client.
 ssl-ca = ""

pkg/config/config_test.go

@@ -126,7 +126,7 @@ func TestLogConfig(t *testing.T) {
 		require.Equal(t, expectedDisableErrorStack, conf.Log.DisableErrorStack)
 		require.Equal(t, expectedEnableTimestamp, conf.Log.EnableTimestamp)
 		require.Equal(t, expectedDisableTimestamp, conf.Log.DisableTimestamp)
-		require.Equal(t, logutil.NewLogConfig("info", "text", "tidb-slow.log", conf.Log.File, resultedDisableTimestamp, func(config *zaplog.Config) { config.DisableErrorVerbose = resultedDisableErrorVerbose }), conf.Log.ToLogConfig())
+		require.Equal(t, logutil.NewLogConfig("info", "text", "tidb-slow.log", "", conf.Log.File, resultedDisableTimestamp, func(config *zaplog.Config) { config.DisableErrorVerbose = resultedDisableErrorVerbose }), conf.Log.ToLogConfig())
 		err := f.Truncate(0)
 		require.NoError(t, err)
 		_, err = f.Seek(0, 0)
@@ -910,7 +910,7 @@ spilled-file-encryption-method = "aes128-ctr"
 	require.Equal(t, GetGlobalConfig(), conf)
 
 	// Test for log config.
-	require.Equal(t, logutil.NewLogConfig("info", "text", "tidb-slow.log", conf.Log.File, false, func(config *zaplog.Config) { config.DisableErrorVerbose = conf.Log.getDisableErrorStack() }), conf.Log.ToLogConfig())
+	require.Equal(t, logutil.NewLogConfig("info", "text", "tidb-slow.log", "", conf.Log.File, false, func(config *zaplog.Config) { config.DisableErrorVerbose = conf.Log.getDisableErrorStack() }), conf.Log.ToLogConfig())
 
 	// Test for tracing config.
 	tracingConf := &tracing.Configuration{

pkg/executor/cluster_table_test.go

@@ -15,10 +15,12 @@
 package executor_test
 
 import (
+	"compress/gzip"
 	"context"
 	"fmt"
 	"net"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -208,71 +210,78 @@ select 10;`
 	fileName2 := "tidb-slow-20236-2020-02-16T19-04-05.01.log"
 	fileName3 := "tidb-slow-20236-2020-02-17T18-00-05.01.log"
 	fileName4 := "tidb-slow-20236.log"
-	fileNames := []string{fileName0, fileName1, fileName2, fileName3, fileName4}
 	defer config.RestoreFunc()()
 	config.UpdateGlobal(func(conf *config.Config) {
 		conf.Log.SlowQueryFile = fileName4
 	})
-	prepareLogs(t, logData, fileNames)
-	defer func() {
-		removeFiles(t, fileNames)
-	}()
-	tk := testkit.NewTestKit(t, store)
-	loc, err := time.LoadLocation("Asia/Shanghai")
-	require.NoError(t, err)
-	tk.Session().GetSessionVars().TimeZone = loc
-	tk.MustExec("use information_schema")
-	cases := []struct {
-		prepareSQL string
-		sql        string
-		result     []string
-	}{
-		{
-			prepareSQL: "set @@time_zone = '+08:00'",
-			sql:        "select time from cluster_slow_query where time > '2020-02-17 12:00:05.000000' and time < '2020-05-14 20:00:00.000000'",
-			result:     []string{"2020-02-17 18:00:05.000000", "2020-02-17 19:00:00.000000", "2020-05-14 19:03:54.314615"},
-		},
-		{
-			prepareSQL: "set @@time_zone = '+08:00'",
-			sql:        "select time from cluster_slow_query where time > '2020-02-17 12:00:05.000000' and time < '2020-05-14 20:00:00.000000' order by time desc",
-			result:     []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000"},
-		},
-		{
-			prepareSQL: "set @@time_zone = '+08:00'",
-			sql:        "select time from cluster_slow_query where (time > '2020-02-15 18:00:00' and time < '2020-02-15 20:01:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-14 20:00:00') order by time",
-			result:     []string{"2020-02-15 18:00:01.000000", "2020-02-15 19:00:05.000000", "2020-02-17 18:00:05.000000", "2020-02-17 19:00:00.000000", "2020-05-14 19:03:54.314615"},
-		},
-		{
-			prepareSQL: "set @@time_zone = '+08:00'",
-			sql:        "select time from cluster_slow_query where (time > '2020-02-15 18:00:00' and time < '2020-02-15 20:01:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-14 20:00:00') order by time desc",
-			result:     []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000", "2020-02-15 19:00:05.000000", "2020-02-15 18:00:01.000000"},
-		},
-		{
-			prepareSQL: "set @@time_zone = '+08:00'",
-			sql:        "select count(*) from cluster_slow_query where time > '2020-02-15 18:00:00.000000' and time < '2020-05-14 20:00:00.000000' order by time desc",
-			result:     []string{"9"},
-		},
-		{
-			prepareSQL: "set @@time_zone = '+08:00'",
-			sql:        "select count(*) from cluster_slow_query where (time > '2020-02-16 18:00:00' and time < '2020-05-14 20:00:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-17 20:00:00')",
-			result:     []string{"6"},
-		},
-		{
-			prepareSQL: "set @@time_zone = '+08:00'",
-			sql:        "select count(*) from cluster_slow_query where time > '2020-02-16 18:00:00.000000' and time < '2020-02-17 20:00:00.000000' order by time desc",
-			result:     []string{"5"},
-		},
-		{
-			prepareSQL: "set @@time_zone = '+08:00'",
-			sql:        "select time from cluster_slow_query where time > '2020-02-16 18:00:00.000000' and time < '2020-05-14 20:00:00.000000' order by time desc limit 3",
-			result:     []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000"},
-		},
-	}
-	for _, cas := range cases {
-		if len(cas.prepareSQL) > 0 {
-			tk.MustExec(cas.prepareSQL)
+	for k := 0; k < 2; k++ {
+		// k = 0 for normal files
+		// k = 1 for compressed files
+		var fileNames []string
+		if k == 0 {
+			fileNames = []string{fileName0, fileName1, fileName2, fileName3, fileName4}
+		} else {
+			fileNames = []string{fileName0 + ".gz", fileName1 + ".gz", fileName2 + ".gz", fileName3 + ".gz", fileName4}
+		}
+		prepareLogs(t, logData, fileNames)
+		tk := testkit.NewTestKit(t, store)
+		loc, err := time.LoadLocation("Asia/Shanghai")
+		require.NoError(t, err)
+		tk.Session().GetSessionVars().TimeZone = loc
+		tk.MustExec("use information_schema")
+		cases := []struct {
+			prepareSQL string
+			sql        string
+			result     []string
+		}{
+			{
+				prepareSQL: "set @@time_zone = '+08:00'",
+				sql:        "select time from cluster_slow_query where time > '2020-02-17 12:00:05.000000' and time < '2020-05-14 20:00:00.000000'",
+				result:     []string{"2020-02-17 18:00:05.000000", "2020-02-17 19:00:00.000000", "2020-05-14 19:03:54.314615"},
+			},
+			{
+				prepareSQL: "set @@time_zone = '+08:00'",
+				sql:        "select time from cluster_slow_query where time > '2020-02-17 12:00:05.000000' and time < '2020-05-14 20:00:00.000000' order by time desc",
+				result:     []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000"},
+			},
+			{
+				prepareSQL: "set @@time_zone = '+08:00'",
+				sql:        "select time from cluster_slow_query where (time > '2020-02-15 18:00:00' and time < '2020-02-15 20:00:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-14 20:00:00') order by time",
+				result:     []string{"2020-02-15 18:00:01.000000", "2020-02-15 19:00:05.000000", "2020-02-17 18:00:05.000000", "2020-02-17 19:00:00.000000", "2020-05-14 19:03:54.314615"},
+			},
+			{
+				prepareSQL: "set @@time_zone = '+08:00'",
+				sql:        "select time from cluster_slow_query where (time > '2020-02-15 18:00:00' and time < '2020-02-15 20:00:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-14 20:00:00') order by time desc",
+				result:     []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000", "2020-02-15 19:00:05.000000", "2020-02-15 18:00:01.000000"},
+			},
+			{
+				prepareSQL: "set @@time_zone = '+08:00'",
+				sql:        "select count(*) from cluster_slow_query where time > '2020-02-15 18:00:00.000000' and time < '2020-05-14 20:00:00.000000' order by time desc",
+				result:     []string{"9"},
+			},
+			{
+				prepareSQL: "set @@time_zone = '+08:00'",
+				sql:        "select count(*) from cluster_slow_query where (time > '2020-02-16 18:00:00' and time < '2020-05-14 20:00:00') or (time > '2020-02-17 18:00:00' and time < '2020-05-17 20:00:00')",
+				result:     []string{"6"},
+			},
+			{
+				prepareSQL: "set @@time_zone = '+08:00'",
+				sql:        "select count(*) from cluster_slow_query where time > '2020-02-16 18:00:00.000000' and time < '2020-02-17 20:00:00.000000' order by time desc",
+				result:     []string{"5"},
+			},
+			{
+				prepareSQL: "set @@time_zone = '+08:00'",
+				sql:        "select time from cluster_slow_query where time > '2020-02-16 18:00:00.000000' and time < '2020-05-14 20:00:00.000000' order by time desc limit 3",
+				result:     []string{"2020-05-14 19:03:54.314615", "2020-02-17 19:00:00.000000", "2020-02-17 18:00:05.000000"},
+			},
+		}
+		for _, cas := range cases {
+			if len(cas.prepareSQL) > 0 {
+				tk.MustExec(cas.prepareSQL)
+			}
+			tk.MustQuery(cas.sql).Check(testkit.RowsWithSep("|", cas.result...))
 		}
-		tk.MustQuery(cas.sql).Check(testkit.RowsWithSep("|", cas.result...))
+		removeFiles(t, fileNames)
 	}
 }
 
@@ -308,12 +317,26 @@ func TestSQLDigestTextRetriever(t *testing.T) {
 }
 
 func prepareLogs(t *testing.T, logData []string, fileNames []string) {
+	writeFile := func(file string, data string) {
+		if strings.HasSuffix(file, ".gz") {
+			f, err := os.Create(file)
+			require.NoError(t, err)
+			gz := gzip.NewWriter(f)
+			_, err = gz.Write([]byte(data))
+			require.NoError(t, err)
+			require.NoError(t, gz.Close())
+			require.NoError(t, f.Close())
+		} else {
+			f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+			require.NoError(t, err)
+			_, err = f.Write([]byte(data))
+			require.NoError(t, err)
+			require.NoError(t, f.Close())
+		}
+	}
+
 	for i, log := range logData {
-		f, err := os.OpenFile(fileNames[i], os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
-		require.NoError(t, err)
-		_, err = f.Write([]byte(log))
-		require.NoError(t, err)
-		require.NoError(t, f.Close())
+		writeFile(fileNames[i], log)
 	}
 }