diff --git a/README.md b/README.md index 2d1dd14..6e865e4 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,16 @@ plugins: - serverless-s3-sync ``` +### Compatibility with Serverless Framework + +Version 2.0.0 is compatible with Serverless Framework v3, but it uses the legacy logging interface. Version 3.0.0 and later uses the [new logging interface](https://www.serverless.com/framework/docs/guides/plugins/cli-output). + +|serverless-s3-sync|Serverless Framework| +|---|---| +|v1.x|v1.x, v2.x| +|v2.0.0|v1.x, v2.x, v3.x| +|≥ v3.0.0|v3.x| + ## Setup ```yaml diff --git a/index.js b/index.js index 3eee99a..c304397 100644 --- a/index.js +++ b/index.js @@ -2,22 +2,22 @@ const BbPromise = require('bluebird'); const s3 = require('@auth0/s3'); -const chalk = require('chalk'); const minimatch = require('minimatch'); const path = require('path'); const fs = require('fs'); const resolveStackOutput = require('./resolveStackOutput') const getAwsOptions = require('./getAwsOptions') -const messagePrefix = 'S3 Sync: '; const mime = require('mime'); const child_process = require('child_process'); const toS3Path = (osPath) => osPath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'); class ServerlessS3Sync { - constructor(serverless, options) { + constructor(serverless, options, logging) { this.serverless = serverless; this.options = options || {}; + this.log = logging.log; + this.progress = logging.progress; this.servicePath = this.serverless.service.serverless.config.servicePath; this.commands = { @@ -71,12 +71,12 @@ class ServerlessS3Sync { 'after:offline:start:init': () => noSync ? undefined : BbPromise.bind(this).then(this.sync).then(this.syncMetadata).then(this.syncBucketTags), 'after:offline:start': () => noSync ? undefined : BbPromise.bind(this).then(this.sync).then(this.syncMetadata).then(this.syncBucketTags), 'before:remove:remove': () => noSync ? undefined : BbPromise.bind(this).then(this.clear), - 's3sync:sync': () => BbPromise.bind(this).then(this.sync), - 's3sync:metadata': () => BbPromise.bind(this).then(this.syncMetadata), - 's3sync:tags': () => BbPromise.bind(this).then(this.syncBucketTags), - 's3sync:bucket:sync': () => BbPromise.bind(this).then(this.sync), - 's3sync:bucket:metadata': () => BbPromise.bind(this).then(this.syncMetadata), - 's3sync:bucket:tags': () => BbPromise.bind(this).then(this.syncBucketTags), + 's3sync:sync': () => BbPromise.bind(this).then(() => this.sync(true)), + 's3sync:metadata': () => BbPromise.bind(this).then(() => this.syncMetadata(true)), + 's3sync:tags': () => BbPromise.bind(this).then(() => this.syncBucketTags(true)), + 's3sync:bucket:sync': () => BbPromise.bind(this).then(() => this.sync(true)), + 's3sync:bucket:metadata': () => BbPromise.bind(this).then(() => this.syncMetadata(true)), + 's3sync:bucket:tags': () => BbPromise.bind(this).then(() => this.syncBucketTags(true)), }; } @@ -112,21 +112,22 @@ class ServerlessS3Sync { return s3.createClient({ s3Client }); } - sync() { + sync(invokedAsCommand) { let s3Sync = this.serverless.service.custom.s3Sync; if(s3Sync.hasOwnProperty('buckets')) { s3Sync = s3Sync.buckets; } - const cli = this.serverless.cli; if (!Array.isArray(s3Sync)) { - cli.consoleLog(`${messagePrefix}${chalk.red('No configuration found')}`) + this.log.error('serverless-s3-sync requires at least one configuration entry in custom.s3Sync') return Promise.resolve(); } - if (this.options.bucket) { - cli.consoleLog(`${messagePrefix}${chalk.yellow(`Syncing directory attached to S3 bucket ${this.options.bucket}...`)}`); - } else { - cli.consoleLog(`${messagePrefix}${chalk.yellow('Syncing directories and S3 prefixes...')}`); - } + + const taskProgress = this.progress.create({ + message: this.options.bucket? + `Syncing directory attached to S3 bucket ${this.options.bucket}` : + 'Syncing directories to S3 buckets' + }) + const servicePath = this.servicePath; const promises = s3Sync.map((s) => { let bucketPrefix = ''; @@ -167,8 +168,13 @@ class ServerlessS3Sync { return new Promise((resolve) => { const localDir = [servicePath, s.localDir].join('/'); + // we're doing the upload in parallel for all buckets, so create one progress entry for each + let percent = 0; + const getProgressMessage = () => `${localDir}: sync with bucket ${bucketName} (${percent}%)`; + const bucketProgress = this.progress.create({ message: getProgressMessage() }) + if (typeof(preCommand) != 'undefined') { - cli.consoleLog(`${messagePrefix}${chalk.yellow('Running pre-command...')}`); + bucketProgress.update(`${localDir}: running pre-command...`); child_process.execSync(preCommand, { stdio: 'inherit' }); } @@ -208,11 +214,14 @@ class ServerlessS3Sync { if (typeof(defaultContentType) != 'undefined') { Object.assign(params, {defaultContentType: defaultContentType}) } + + bucketProgress.update(getProgressMessage()); + const uploader = this.client().uploadDir(params); uploader.on('error', (err) => { + bucketProgress.remove(); throw err; }); - let percent = 0; uploader.on('progress', () => { if (uploader.progressTotal === 0) { return; @@ -220,10 +229,11 @@ class ServerlessS3Sync { const current = Math.round((uploader.progressAmount / uploader.progressTotal) * 10) * 10; if (current > percent) { percent = current; - cli.printDot(); + bucketProgress.update(getProgressMessage()); } }); uploader.on('end', () => { + bucketProgress.remove(); resolve('done'); }); }); @@ -231,9 +241,14 @@ class ServerlessS3Sync { }); return Promise.all(promises) .then(() => { - cli.printDot(); - cli.consoleLog(''); - cli.consoleLog(`${messagePrefix}${chalk.yellow('Synced.')}`); + if (invokedAsCommand) { + this.log.success('Synced files to S3 buckets'); + } else { + this.log.verbose('Synced files to S3 buckets'); + } + }) + .finally(() => { + taskProgress.remove(); }); } @@ -243,10 +258,12 @@ class ServerlessS3Sync { s3Sync = s3Sync.buckets; } if (!Array.isArray(s3Sync)) { + this.log.notice(`No configuration found for serverless-s3-sync, skipping removal...`); return Promise.resolve(); } - const cli = this.serverless.cli; - cli.consoleLog(`${messagePrefix}${chalk.yellow('Removing S3 objects...')}`); + + const taskProgress = this.progress.create({ message: 'Removing objects from S3 buckets' }); + const promises = s3Sync.map((s) => { let bucketPrefix = ''; if (s.hasOwnProperty('bucketPrefix')) { @@ -259,11 +276,16 @@ class ServerlessS3Sync { Bucket: bucketName, Prefix: bucketPrefix }; + + let percent = 0; + let getProgressMessage = () => `${bucketName}: removing files with prefix ${bucketPrefix} (${percent}%)` + const bucketProgress = this.progress.create({ message: getProgressMessage() }) + const uploader = this.client().deleteDir(params); uploader.on('error', (err) => { + bucketProgress.remove(); throw err; }); - let percent = 0; uploader.on('progress', () => { if (uploader.progressTotal === 0) { return; @@ -271,34 +293,37 @@ class ServerlessS3Sync { const current = Math.round((uploader.progressAmount / uploader.progressTotal) * 10) * 10; if (current > percent) { percent = current; - cli.printDot(); + bucketProgress.update(getProgressMessage()); } }); uploader.on('end', () => { + bucketProgress.remove(); resolve('done'); }); }); }); }); - return Promise.all(promises) + return Promise.all((promises)) .then(() => { - cli.printDot(); - cli.consoleLog(''); - cli.consoleLog(`${messagePrefix}${chalk.yellow('Removed.')}`); + this.log.verbose('Removed objects from S3 buckets'); + }) + .finally(() => { + taskProgress.remove(); }); } - syncMetadata() { + syncMetadata(invokedAsCommand) { let s3Sync = this.serverless.service.custom.s3Sync; if(s3Sync.hasOwnProperty('buckets')) { s3Sync = s3Sync.buckets; } - const cli = this.serverless.cli; if (!Array.isArray(s3Sync)) { - cli.consoleLog(`${messagePrefix}${chalk.red('No configuration found')}`) + this.log.error('serverless-s3-sync requires at least one configuration entry in custom.s3Sync'); return Promise.resolve(); } - cli.consoleLog(`${messagePrefix}${chalk.yellow('Syncing metadata...')}`); + + const taskProgress = this.progress.create({ message: 'Syncing bucket metadata' }); + const servicePath = this.servicePath; const promises = s3Sync.map( async (s) => { let bucketPrefix = ''; @@ -342,7 +367,13 @@ class ServerlessS3Sync { return null; } - return Promise.all(filesToSync.map((file) => { + const bucketDir = `${bucketName}${bucketPrefix == '' ? '' : bucketPrefix}/`; + + let percent = 0; + const getProgressMessage = () => `${localDir}: sync bucket metadata to ${bucketDir} (${percent}%)` + const bucketProgress = this.progress.create({ message: getProgressMessage() }) + + return Promise.all(filesToSync.map((file, index) => { return new Promise((resolve) => { let contentTypeObject = {}; let detectedContentType = mime.getType(file.name) @@ -353,7 +384,7 @@ class ServerlessS3Sync { ...contentTypeObject, ...file.params, ...{ - CopySource: toS3Path(file.name.replace(path.resolve(localDir) + path.sep, `${bucketName}${bucketPrefix == '' ? '' : bucketPrefix}/`)), + CopySource: toS3Path(file.name.replace(path.resolve(localDir) + path.sep, bucketDir)), Key: toS3Path(file.name.replace(path.resolve(localDir) + path.sep, `${bucketPrefix ? bucketPrefix.replace(/^\//, '') + '/' : ''}`)), Bucket: bucketName, ACL: acl, @@ -365,31 +396,43 @@ class ServerlessS3Sync { throw err; }); uploader.on('end', () => { + const current = Math.round((index / filesToSync.length) * 10) * 10; + if (current > percent) { + percent = current; + bucketProgress.update(getProgressMessage()) + } resolve('done'); }); }); - })); + })).finally(() => { + bucketProgress.remove(); + }); }); }); return Promise.all((promises)) .then(() => { - cli.printDot(); - cli.consoleLog(''); - cli.consoleLog(`${messagePrefix}${chalk.yellow('Synced metadata.')}`); + if (invokedAsCommand) { + this.log.success('Synced bucket metadata'); + } else { + this.log.verbose('Synced bucket metadata'); + } + }) + .finally(() => { + taskProgress.remove(); }); } - syncBucketTags() { + syncBucketTags(invokedAsCommand) { let s3Sync = this.serverless.service.custom.s3Sync; if(s3Sync.hasOwnProperty('buckets')) { s3Sync = s3Sync.buckets; } - const cli = this.serverless.cli; if (!Array.isArray(s3Sync)) { - cli.consoleLog(`${messagePrefix}${chalk.red('No configuration found')}`) + this.log.error('serverless-s3-sync requires at least one configuration entry in custom.s3Sync'); return Promise.resolve(); } - cli.consoleLog(`${messagePrefix}${chalk.yellow('Updating bucket tags...')}`); + + const taskProgress = this.progress.create({ message: 'Updating bucket tags' }); const promises = s3Sync.map( async (s) => { if (!s.bucketName && !s.bucketNameKey) { @@ -416,6 +459,8 @@ class ServerlessS3Sync { return null; } + const bucketProgress = this.progress.create({ message: `${bucketName}: sync bucket tags` }) + // AWS.S3 does not have an option to append tags to a bucket, it can only rewrite the whole set of tags // To avoid removing system tags set by other tools, we read the existing tags, merge our tags in the list // and then write them all back @@ -432,14 +477,22 @@ class ServerlessS3Sync { }; return this.client().s3.putBucketTagging(putParams).promise(); }) + .finally(() => { + bucketProgress.remove(); + }); }); }); return Promise.all((promises)) .then(() => { - cli.printDot(); - cli.consoleLog(''); - cli.consoleLog(`${messagePrefix}${chalk.yellow('Updated bucket tags.')}`); + if (invokedAsCommand) { + this.log.success('Updated bucket tags'); + } else { + this.log.verbose('Updated bucket tags'); + } + }) + .finally(() => { + taskProgress.remove(); }); } @@ -455,11 +508,10 @@ class ServerlessS3Sync { } getLocalFiles(dir, files) { - const cli = this.serverless.cli; try { fs.accessSync(dir, fs.constants.R_OK); } catch (e) { - cli.consoleLog(`${messagePrefix}${chalk.red(`The directory ${dir} does not exist.`)}`); + this.log.error(`The directory ${dir} does not exist.`); return files; } fs.readdirSync(dir).forEach(file => { @@ -467,7 +519,7 @@ class ServerlessS3Sync { try { fs.accessSync(fullPath, fs.constants.R_OK); } catch (e) { - cli.consoleLog(`${messagePrefix}${chalk.red(`The file ${fullPath} doesn not exist.`)}`); + this.log.error(`The file ${fullPath} does not exist.`); return; } if (fs.lstatSync(fullPath).isDirectory()) { diff --git a/package-lock.json b/package-lock.json index 317cb7f..faf23f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "serverless-s3-sync", - "version": "1.12.0", + "version": "2.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -20,14 +20,6 @@ "streamsink": "~1.2.0" } }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "requires": { - "color-convert": "^1.9.0" - } - }, "aws-sdk": { "version": "2.441.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.441.0.tgz", @@ -78,39 +70,11 @@ "isarray": "^1.0.0" } }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" - }, "events": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", @@ -134,11 +98,6 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" - }, "ieee754": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", @@ -210,14 +169,6 @@ "resolved": "https://registry.npmjs.org/streamsink/-/streamsink-1.2.0.tgz", "integrity": "sha1-76/unx4i01ke1949yqlcP1559zw=" }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "requires": { - "has-flag": "^3.0.0" - } - }, "url": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", diff --git a/package.json b/package.json index c39850f..4c931ec 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,10 @@ "dependencies": { "@auth0/s3": "^1.0.0", "bluebird": "^3.5.4", - "chalk": "^2.4.2", "mime": "^2.4.4", "minimatch": "^3.0.4" + }, + "peerDependencies": { + "serverless": "^3.0.0" } }