From 13942ec0140247b1272e4832ed388ea4c2ce7ed4 Mon Sep 17 00:00:00 2001 From: lydoan1199 Date: Tue, 26 Dec 2023 17:20:00 +0700 Subject: [PATCH 1/4] override apikey in command line --- package.json | 2 +- src/core/utils.js | 18 ++++++++++++++++++ src/service/agent.js | 8 ++++---- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 236699a1..294011cd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "katalon-agent", - "version": "v2.0.0", + "version": "v2.0.1", "description": "", "main": "cli.js", "scripts": { diff --git a/src/core/utils.js b/src/core/utils.js index da66a8a5..12953378 100644 --- a/src/core/utils.js +++ b/src/core/utils.js @@ -41,6 +41,24 @@ module.exports = { }, command); }, + overrideCommand(command, ...options) { + return options.reduce((cmd, option) => { + const { flag, value } = option; + if (value) { + if (cmd.includes(flag)) { + const arStr = cmd.split(' '); + const index = arStr.findIndex((t) => t.startsWith(flag)); + if (index >= 0) { + arStr[index] = arStr[index].replace(/=.*/, `="${value}"`); + return arStr.join(' '); + } + } + return `${cmd} ${flag}="${value}"`; + } + return cmd; + }, command); + }, + getVersion() { return packageJson.version; }, diff --git a/src/service/agent.js b/src/service/agent.js index 891e31b9..06fb80cf 100644 --- a/src/service/agent.js +++ b/src/service/agent.js @@ -301,12 +301,12 @@ class Agent { let ksArgs; if (config.isOnPremise) { - ksArgs = utils.updateCommand(parameter.command, { + ksArgs = utils.overrideCommand(parameter.command, { flag: '-apiKeyOnPremise', value: apiKey, }); } else { - ksArgs = utils.updateCommand( + ksArgs = utils.overrideCommand( parameter.command, { flag: '-apiKey', value: apiKey }, ); @@ -394,12 +394,12 @@ class Agent { let ksArgs; if (config.isOnPremise) { - ksArgs = utils.updateCommand(parameter.command, { + ksArgs = utils.overrideCommand(parameter.command, { flag: '-apiKeyOnPremise', value: apiKey, }); } else { - ksArgs = utils.updateCommand( + ksArgs = utils.overrideCommand( parameter.command, { flag: '-apiKey', value: apiKey }, ); From 34f28c4845dc546f09c4565ddac677c0d573ec31 Mon Sep 17 00:00:00 2001 From: "hieu.mnguyen" Date: Wed, 3 Jan 2024 12:19:35 +0700 Subject: [PATCH 2/4] [TES-417]: impl refresh presign-url --- src/config/transports.js | 32 ++++++++++++++++++++++++++++++++ src/service/agent.js | 2 ++ 2 files changed, 34 insertions(+) diff --git a/src/config/transports.js b/src/config/transports.js index c31c5817..1e229ba7 100644 --- a/src/config/transports.js +++ b/src/config/transports.js @@ -1,11 +1,16 @@ const _ = require('lodash'); +const path = require('path'); +const moment = require('moment'); const TransportStream = require('winston-transport'); +const { generateUuid } = require('../helper/agent'); const api = require('../core/api'); class S3FileTransport extends TransportStream { constructor(options = {}, afterLog) { super(options); + this.jobInfo = options.jobInfo; + this.apiKey = options.apiKey; this.filePath = options.filePath; this.signedUrl = options.signedUrl; this.parentLogger = options.logger; @@ -18,6 +23,33 @@ class S3FileTransport extends TransportStream { } uploadToS3() { + const parsedUrl = new URL(this.signedUrl); + const params = new URLSearchParams(parsedUrl.search); + + const amzDate = params.get('X-Amz-Date'); + const amzExpires = params.get('X-Amz-Expires'); + + const dateFormart = 'YYYYMMDDTHHmmss[Z]'; + const dateExpires = moment.utc(amzDate, dateFormart).toDate(); + dateExpires.setSeconds(dateExpires.getSeconds(), amzExpires * 1000); + + const now = new Date(); + if (dateExpires < now) { + const requestGetUploadInfo = async () => { + const response = await api.getUploadInfo(this.jobInfo.projectId, this.apiKey); + if (!response || !response.body) { + return null; + } + const { body } = response; + const { uploadUrl } = body; + this.signedUrl = uploadUrl; + const batch = generateUuid(); + const fileName = path.basename(this.filePath); + return api.saveJobLog(this.jobInfo, batch, fileName, this.apiKey); + }; + requestGetUploadInfo(); + } + return api .uploadFile(this.signedUrl, this.filePath) .then(() => this.afterLog && this.afterLog()) diff --git a/src/service/agent.js b/src/service/agent.js index 06fb80cf..7eb77844 100644 --- a/src/service/agent.js +++ b/src/service/agent.js @@ -156,6 +156,8 @@ async function executeJob(jobInfo, keepFiles) { jLogger.add( new S3FileTransport( { + jobInfo, + apiKey, filePath: logFilePath, signedUrl: jobInfo.uploadUrl, logger, From 4e5da49bfcaa4b4bcb226238960c6f94bf88bbd1 Mon Sep 17 00:00:00 2001 From: "hieu.mnguyen" Date: Wed, 3 Jan 2024 16:15:39 +0700 Subject: [PATCH 3/4] [TES-417] added async await --- src/config/transports.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/transports.js b/src/config/transports.js index 1e229ba7..8a274fc4 100644 --- a/src/config/transports.js +++ b/src/config/transports.js @@ -22,7 +22,7 @@ class S3FileTransport extends TransportStream { this.uploadToS3Throttled = _.throttle(this.uploadToS3, this.wait, { trailing: false }); } - uploadToS3() { + async uploadToS3() { const parsedUrl = new URL(this.signedUrl); const params = new URLSearchParams(parsedUrl.search); @@ -47,7 +47,7 @@ class S3FileTransport extends TransportStream { const fileName = path.basename(this.filePath); return api.saveJobLog(this.jobInfo, batch, fileName, this.apiKey); }; - requestGetUploadInfo(); + await requestGetUploadInfo(); } return api From 7387d8fcc0869d0d09f62af6a5569c287a60bd8d Mon Sep 17 00:00:00 2001 From: "hieu.mnguyen" Date: Tue, 9 Jan 2024 17:54:20 +0700 Subject: [PATCH 4/4] [TES-417] add more 2 minutes to ensure regenerate presinged url --- src/config/transports.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config/transports.js b/src/config/transports.js index 8a274fc4..d4410659 100644 --- a/src/config/transports.js +++ b/src/config/transports.js @@ -32,6 +32,8 @@ class S3FileTransport extends TransportStream { const dateFormart = 'YYYYMMDDTHHmmss[Z]'; const dateExpires = moment.utc(amzDate, dateFormart).toDate(); dateExpires.setSeconds(dateExpires.getSeconds(), amzExpires * 1000); + // Minus more 2 minutes to ensure regenerate presigned url + dateExpires.setSeconds(dateExpires.getSeconds(), -(2 * 60 * 1000)); const now = new Date(); if (dateExpires < now) {