feat: support gzip compress on rotate file (#30)

[skip ci]

---------

Co-authored-by: cuichuanteng <[email protected]>
Co-authored-by: fengmk2 <[email protected]>

README.md

@@ -41,6 +41,7 @@ exports.logrotator = {
   maxFiles: 10,                    // pieces rotate by size
   rotateDuration: 60000,           // time interval to judge if any file need rotate
   maxDays: 31,                     // keep max days log files, default is `31`. Set `0` to keep all logs
+  gzip:false,                      // use gzip compress logger on rotate file, default is `false`. Set `true` to enable
 };
 ```
 

app/lib/day_rotator.js

@@ -74,11 +74,12 @@ class DayRotator extends Rotator {
     }
 
     if (!files.has(srcPath)) {
+      const ext = this.app.config.logrotator.gzip === true ? '.gz' : '';
       // allow 2 minutes deviation
       const targetPath = srcPath + moment()
         .subtract(23, 'hours')
         .subtract(58, 'minutes')
-        .format('.YYYY-MM-DD');
+        .format('.YYYY-MM-DD') + ext;
       debug('set file %s => %s', srcPath, targetPath);
       files.set(srcPath, { srcPath, targetPath });
     }

app/lib/hour_rotator.js

@@ -35,7 +35,8 @@ class DayRotator extends Rotator {
 
   _setFile(srcPath, files) {
     if (!files.has(srcPath)) {
-      const targetPath = srcPath + moment().subtract(1, 'hours').format(`.YYYY-MM-DD${this.hourDelimiter}HH`);
+      const ext = this.app.config.logrotator.gzip === true ? '.gz' : '';
+      const targetPath = srcPath + moment().subtract(1, 'hours').format(`.YYYY-MM-DD${this.hourDelimiter}HH`) + ext;
       debug('set file %s => %s', srcPath, targetPath);
       files.set(srcPath, { srcPath, targetPath });
     }

app/lib/rotator.js

@@ -1,10 +1,12 @@
 'use strict';
 
 const assert = require('assert');
+const { createWriteStream, createReadStream } = require('fs');
 const fs = require('mz/fs');
+const { pipeline } = require('stream');
+const { createGzip } = require('zlib');
 const debug = require('util').debuglog('egg-logrotator:rotator');
 
-
 class Rotator {
 
   constructor(options) {
@@ -25,7 +27,7 @@ class Rotator {
     for (const file of files.values()) {
       try {
         debug('rename from %s to %s', file.srcPath, file.targetPath);
-        await renameOrDelete(file.srcPath, file.targetPath);
+        await renameOrDelete(file.srcPath, file.targetPath, this.app.config.logrotator.gzip);
         rotatedFile.push(`${file.srcPath} -> ${file.targetPath}`);
       } catch (err) {
         err.message = `[egg-logrotator] rename ${file.srcPath}, found exception: ` + err.message;
@@ -48,7 +50,7 @@ class Rotator {
 module.exports = Rotator;
 
 // rename from srcPath to targetPath, for example foo.log.1 > foo.log.2
-async function renameOrDelete(srcPath, targetPath) {
+async function renameOrDelete(srcPath, targetPath, gzip) {
   if (srcPath === targetPath) {
     return;
   }
@@ -63,5 +65,23 @@ async function renameOrDelete(srcPath, targetPath) {
     const err = new Error(`targetFile ${targetPath} exists!!!`);
     throw err;
   }
-  await fs.rename(srcPath, targetPath);
+  // if gzip is true, then use gzip
+  if (gzip === true) {
+    const tmpPath = `${targetPath}.tmp`;
+    await fs.rename(srcPath, tmpPath);
+    await (() => {
+      return new Promise((resolve, reject) => {
+        pipeline(createReadStream(tmpPath), createGzip(), createWriteStream(targetPath), async err => {
+          if (err) {
+            reject(err);
+          } else {
+            await fs.unlink(tmpPath);
+            resolve();
+          }
+        });
+      });
+    })();
+  } else {
+    await fs.rename(srcPath, targetPath);
+  }
 }

app/lib/size_rotator.js

@@ -50,17 +50,18 @@ class SizeRotator extends Rotator {
     if (files.has(logPath)) {
       return;
     }
+    const ext = this.app.config.logrotator.gzip === true ? '.gz' : '';
     // foo.log.2 -> foo.log.3
     // foo.log.1 -> foo.log.2
     for (let i = maxFiles - 1; i >= 1; i--) {
       const srcPath = `${logPath}.${i}`;
-      const targetPath = `${logPath}.${i + 1}`;
+      const targetPath = `${logPath}.${i + 1}${ext}`;
       debug('set file %s => %s', srcPath, targetPath);
       files.set(srcPath, { srcPath, targetPath });
     }
     // foo.log -> foo.log.1
     debug('set file %s => %s', logPath, `${logPath}.1`);
-    files.set(logPath, { srcPath: logPath, targetPath: `${logPath}.1` });
+    files.set(logPath, { srcPath: logPath, targetPath: `${logPath}.1${ext}` });
   }
 
 }

config/config.default.js

@@ -22,4 +22,6 @@ exports.logrotator = {
   rotateDuration: 60000,
   // for clean_log
   maxDays: 31,
+  // enable gzip
+  gzip: false,
 };

test/fixtures/logrotator-app-day-gzip/agent.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function(agent) {
+  agent.messenger.on('log-reload', () => console.log('agent got log-reload'));
+};

test/fixtures/logrotator-app-day-gzip/app.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function(app) {
+  app.messenger.on('log-reload', () => console.log('app got log-reload'));
+};

test/fixtures/logrotator-app-day-gzip/app/router.js

@@ -0,0 +1,7 @@
+'use strict';
+
+module.exports = app => {
+  app.get('/', function* () {
+    this.body = 123;
+  });
+};

test/fixtures/logrotator-app-day-gzip/config/config.default.js

@@ -0,0 +1,11 @@
+'use strict';
+
+
+module.exports = () => {
+  const exports = {
+    logrotator: {
+      gzip: true,
+    },
+  };
+  return exports;
+};

test/fixtures/logrotator-app-day-gzip/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "logrotator-app-day-gzip"
+}

test/fixtures/logrotator-app-hour-gzip/agent.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function(agent) {
+  agent.messenger.on('log-reload', () => console.log('agent got log-reload'));
+};

test/fixtures/logrotator-app-hour-gzip/app.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function(app) {
+  app.messenger.on('log-reload', () => console.log('app got log-reload'));
+};

test/fixtures/logrotator-app-hour-gzip/app/router.js

@@ -0,0 +1,7 @@
+'use strict';
+
+module.exports = app => {
+  app.get('/', function* () {
+    this.body = 123;
+  });
+};

test/fixtures/logrotator-app-hour-gzip/config/config.default.js

@@ -0,0 +1,20 @@
+'use strict';
+
+const path = require('path');
+
+module.exports = appInfo => {
+  const exports = {
+    logrotator: {
+      gzip: true,
+      filesRotateByHour: [
+        path.join(appInfo.baseDir, `logs/${appInfo.name}/egg-web.log`),
+        path.join(appInfo.baseDir, `logs/${appInfo.name}/egg-web.log`),
+        // relative path
+        'egg-web.log',
+        // ignore unexist file
+        path.join(appInfo.baseDir, `logs/${appInfo.name}/no-exist.log`),
+      ],
+    },
+  };
+  return exports;
+};

test/fixtures/logrotator-app-hour-gzip/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "logrotator-app-hour-gzip"
+}

test/fixtures/logrotator-app-size-gzip/agent.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function(agent) {
+  agent.messenger.on('log-reload', () => console.log('agent got log-reload'));
+};

test/fixtures/logrotator-app-size-gzip/app.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function(app) {
+  app.messenger.on('log-reload', () => console.log('app got log-reload'));
+};

test/fixtures/logrotator-app-size-gzip/app/router.js

@@ -0,0 +1,7 @@
+'use strict';
+
+module.exports = app => {
+  app.get('/', function* () {
+    this.body = 123;
+  });
+};

test/fixtures/logrotator-app-size-gzip/config/config.default.js

@@ -0,0 +1,22 @@
+'use strict';
+
+const path = require('path');
+
+module.exports = appInfo => {
+  const exports = {
+    logrotator: {
+      gzip: true,
+      filesRotateBySize: [
+        path.join(appInfo.baseDir, `logs/${appInfo.name}/egg-web.log`),
+        path.join(appInfo.baseDir, `logs/${appInfo.name}/egg-web.log`),
+        'egg-web.log',
+        // ignore unexist file
+        path.join(appInfo.baseDir, `logs/${appInfo.name}/no-exist.log`),
+      ],
+      maxFileSize: 1,
+      maxFiles: 2,
+      rotateDuration: 60000,
+    },
+  };
+  return exports;
+};

test/fixtures/logrotator-app-size-gzip/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "logrotator-app-size-gzip"
+}

test/logrotator.test.js

@@ -7,6 +7,7 @@ const fs = require('fs');
 const glob = require('glob');
 const moment = require('moment');
 const assert = require('assert');
+const { createUnzip } = require('zlib');
 
 
 describe('test/logrotator.test.js', () => {
@@ -435,6 +436,94 @@ describe('test/logrotator.test.js', () => {
     });
   });
 
+  describe('rotate_by_hour_gzip', () => {
+    let app;
+    const schedule = path.join(__dirname, '../app/schedule/rotate_by_hour');
+    before(() => {
+      app = mm.app({
+        baseDir: 'logrotator-app-hour-gzip',
+      });
+      return app.ready();
+    });
+    after(() => app.close());
+    afterEach(mm.restore);
+
+    it('should rotate by size and use zlib.gzip compress', function* () {
+      yield app.runSchedule(schedule);
+      yield sleep(100);
+      const logDir = app.config.logger.dir;
+      const date = moment().subtract(1, 'hours').format('YYYY-MM-DD-HH');
+      const file = path.join(logDir, `egg-web.log.${date}.gz`);
+      assert.equal(fs.existsSync(file), true);
+      const gzip = createUnzip();
+      fs.createReadStream(file).pipe(gzip);
+      gzip.on('data', data => {
+        assert(data.toString().includes('logrotator-app-hour-gzip'));
+      });
+    });
+
+  });
+
+  describe('rotate_by_day_gzip', () => {
+    let app;
+    const schedule = path.join(__dirname, '../app/schedule/rotate_by_file');
+    before(() => {
+      app = mm.app({
+        baseDir: 'logrotator-app-day-gzip',
+      });
+      return app.ready();
+    });
+    after(() => app.close());
+    afterEach(mm.restore);
+
+    it('should rotate by size and use zlib.gzip compress', function* () {
+      yield app.runSchedule(schedule);
+      yield sleep(100);
+      const logDir = app.config.logger.dir;
+      const now = moment().startOf('date');
+      const date = now.clone().subtract(1, 'days').format('YYYY-MM-DD');
+      const file = path.join(logDir, `egg-web.log.${date}.gz`);
+      assert.equal(fs.existsSync(file), true);
+      const gzip = createUnzip();
+      fs.createReadStream(file).pipe(gzip);
+      gzip.on('data', data => {
+        assert(data.toString().includes('logrotator-app-day-gzip'));
+      });
+    });
+
+  });
+
+  describe('rotate_by_size_gzip', () => {
+    let mockfile;
+    let app;
+    const schedule = path.join(__dirname, '../app/schedule/rotate_by_size');
+    before(() => {
+      app = mm.app({
+        baseDir: 'logrotator-app-size-gzip',
+      });
+      return app.ready();
+    });
+    before(() => {
+      mockfile = path.join(app.config.logger.dir, 'egg-web.log');
+    });
+    after(() => app.close());
+    afterEach(mm.restore);
+
+    it('should rotate by size', function* () {
+      yield app.runSchedule(schedule);
+      yield sleep(100);
+      const file = `${mockfile}.1.gz`;
+      assert(fs.existsSync(file));
+      const gzip = createUnzip();
+      fs.createReadStream(file).pipe(gzip);
+      gzip.on('data', data => {
+        assert(data.toString().includes('logrotator-app-size-gzip'));
+      });
+
+    });
+
+  });
+
 });
 
 function sleep(ms) {