From 342ddd551deec596a4e53ede81dc95e39680488c Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Wed, 18 Dec 2024 17:56:16 -0800 Subject: [PATCH 1/7] feat: run pushed monitors locally in dry-run mode --- __tests__/push/bundler.test.ts | 5 +- package-lock.json | 111 +++++--------------------- package.json | 4 +- src/dsl/monitor.ts | 9 ++- src/push/bundler.ts | 42 +++++----- src/push/index.ts | 31 +++++--- src/push/monitor.ts | 14 +++- src/push/run-local.ts | 139 +++++++++++++++++++++++++++++++++ src/push/utils.ts | 46 ++++++++++- 9 files changed, 267 insertions(+), 134 deletions(-) create mode 100644 src/push/run-local.ts diff --git a/__tests__/push/bundler.test.ts b/__tests__/push/bundler.test.ts index fe9d34a3..f7636ba1 100644 --- a/__tests__/push/bundler.test.ts +++ b/__tests__/push/bundler.test.ts @@ -25,7 +25,7 @@ import { createReadStream } from 'fs'; import { writeFile, unlink, mkdir, rm } from 'fs/promises'; -import unzipper from 'unzipper'; +import unzip from 'unzip-stream'; import { join } from 'path'; import { generateTempPath } from '../../src/helpers'; import { Bundler } from '../../src/push/bundler'; @@ -74,11 +74,10 @@ async function validateZip(content) { await writeFile(pathToZip, decoded); const files: Array = []; - let contents = ''; await new Promise(r => { createReadStream(pathToZip) - .pipe(unzipper.Parse()) + .pipe(unzip.Parse()) .on('entry', function (entry) { files.push(entry.path); contents += '\n' + entry.path + '\n'; diff --git a/package-lock.json b/package-lock.json index 84cab917..f18b5fce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "source-map-support": "^0.5.21", "stack-utils": "^2.0.6", "undici": "^5.28.3", + "unzip-stream": "^0.3.4", "yaml": "^2.2.2" }, "bin": { @@ -58,8 +59,7 @@ "rimraf": "^3.0.2", "semantic-release": "^21.1.1", "ts-jest": "^29.1.1", - "typescript": "^5.1.6", - "unzipper": "^0.10.11" + "typescript": "^5.1.6" }, "engines": { "node": ">=18.20.3" @@ -5123,20 +5123,10 @@ "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", "dev": true }, - "node_modules/big-integer": { - "version": "1.6.51", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.51.tgz", - "integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==", - "dev": true, - "engines": { - "node": ">=0.6" - } - }, "node_modules/binary": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", - "dev": true, "dependencies": { "buffers": "~0.1.1", "chainsaw": "~0.1.0" @@ -5145,12 +5135,6 @@ "node": "*" } }, - "node_modules/bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", - "dev": true - }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -5244,20 +5228,10 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, - "node_modules/buffer-indexof-polyfill": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", - "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", - "dev": true, - "engines": { - "node": ">=0.10" - } - }, "node_modules/buffers": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", - "dev": true, "engines": { "node": ">=0.2.0" } @@ -5334,7 +5308,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", - "dev": true, "dependencies": { "traverse": ">=0.3.0 <0.4" }, @@ -5343,10 +5316,11 @@ } }, "node_modules/chalk": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", - "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, + "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -5401,10 +5375,11 @@ } }, "node_modules/cli-table3": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.3.tgz", - "integrity": "sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==", + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.5.tgz", + "integrity": "sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==", "dev": true, + "license": "MIT", "dependencies": { "string-width": "^4.2.0" }, @@ -6941,33 +6916,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "dev": true, - "dependencies": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - }, - "engines": { - "node": ">=0.6" - } - }, - "node_modules/fstream/node_modules/rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "dev": true, - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - } - }, "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -9668,12 +9616,6 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, - "node_modules/listenercount": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", - "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=", - "dev": true - }, "node_modules/listr2": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/listr2/-/listr2-3.8.2.tgz", @@ -10157,8 +10099,7 @@ "node_modules/minimist": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", - "dev": true + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/minimist-options": { "version": "4.1.0", @@ -10186,7 +10127,6 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "dev": true, "dependencies": { "minimist": "^1.2.6" }, @@ -15106,12 +15046,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=", - "dev": true - }, "node_modules/sharp": { "version": "0.33.5", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", @@ -15807,7 +15741,6 @@ "version": "0.3.9", "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=", - "dev": true, "engines": { "node": "*" } @@ -16066,22 +15999,14 @@ "node": ">= 10.0.0" } }, - "node_modules/unzipper": { - "version": "0.10.11", - "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz", - "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==", - "dev": true, + "node_modules/unzip-stream": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.4.tgz", + "integrity": "sha512-PyofABPVv+d7fL7GOpusx7eRT9YETY2X04PhwbSipdj6bMxVCFJrr+nm0Mxqbf9hUiTin/UsnuFWBXlDZFy0Cw==", + "license": "MIT", "dependencies": { - "big-integer": "^1.6.17", - "binary": "~0.3.0", - "bluebird": "~3.4.1", - "buffer-indexof-polyfill": "~1.0.0", - "duplexer2": "~0.1.4", - "fstream": "^1.0.12", - "graceful-fs": "^4.2.2", - "listenercount": "~1.0.1", - "readable-stream": "~2.3.6", - "setimmediate": "~1.0.4" + "binary": "^0.3.0", + "mkdirp": "^0.5.1" } }, "node_modules/update-browserslist-db": { diff --git a/package.json b/package.json index a5e431da..65548aa3 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "source-map-support": "^0.5.21", "stack-utils": "^2.0.6", "undici": "^5.28.3", + "unzip-stream": "^0.3.4", "yaml": "^2.2.2" }, "devDependencies": { @@ -96,8 +97,7 @@ "rimraf": "^3.0.2", "semantic-release": "^21.1.1", "ts-jest": "^29.1.1", - "typescript": "^5.1.6", - "unzipper": "^0.10.11" + "typescript": "^5.1.6" }, "overrides": { "follow-redirects": "^1.15.6", diff --git a/src/dsl/monitor.ts b/src/dsl/monitor.ts index 0e2b943c..97470994 100644 --- a/src/dsl/monitor.ts +++ b/src/dsl/monitor.ts @@ -88,7 +88,7 @@ export class Monitor { content?: string; source?: Location; filter: MonitorFilter; - constructor(public config: MonitorConfig = {}) {} + constructor(public config: MonitorConfig = {}) { } /** * Treat the creation time config with `monitor.use` as source of truth by * merging the values coming from CLI and Synthetics config file @@ -158,6 +158,13 @@ export class Monitor { .digest('base64'); } + /** + * Returns the size of the monitor in bytes which is sent as payload to Kibana + */ + size() { + return JSON.stringify(this).length + } + validate() { const schedule = this.config.schedule; if (ALLOWED_SCHEDULES.includes(schedule)) { diff --git a/src/push/bundler.ts b/src/push/bundler.ts index d2b9458d..a3265eff 100644 --- a/src/push/bundler.ts +++ b/src/push/bundler.ts @@ -24,25 +24,22 @@ */ import path from 'path'; -import { stat, unlink, readFile } from 'fs/promises'; +import { unlink, readFile, stat } from 'fs/promises'; import { createWriteStream } from 'fs'; import * as esbuild from 'esbuild'; import archiver from 'archiver'; import { commonOptions } from '../core/transform'; import { SyntheticsBundlePlugin } from './plugin'; -// 1500KB Max Gzipped limit for bundled code to be pushed as Kibana project monitors. -const SIZE_LIMIT_KB = 1500; +// 1500kB Max Gzipped limit for bundled monitor code to be pushed as Kibana project monitors. +const SIZE_LIMIT_KB = 1000; function relativeToCwd(entry: string) { return path.relative(process.cwd(), entry); } export class Bundler { - moduleMap = new Map(); - constructor() {} - - async prepare(absPath: string) { + async bundle(absPath: string) { const options: esbuild.BuildOptions = { ...commonOptions(), ...{ @@ -59,38 +56,36 @@ export class Bundler { if (result.errors.length > 0) { throw result.errors; } - this.moduleMap.set(absPath, result.outputFiles[0].text); + return result.outputFiles[0].text } - async zip(outputPath: string) { + async zip(source: string, code: string, dest: string) { return new Promise((fulfill, reject) => { - const output = createWriteStream(outputPath); + const output = createWriteStream(dest); const archive = archiver('zip', { zlib: { level: 9 }, }); archive.on('error', reject); output.on('close', fulfill); archive.pipe(output); - for (const [path, content] of this.moduleMap.entries()) { - const relativePath = relativeToCwd(path); - // Date is fixed to Unix epoch so the file metadata is - // not modified everytime when files are bundled - archive.append(content, { - name: relativePath, - date: new Date('1970-01-01'), - }); - } + const relativePath = relativeToCwd(source); + // Date is fixed to Unix epoch so the file metadata is + // not modified everytime when files are bundled + archive.append(code, { + name: relativePath, + date: new Date('1970-01-01'), + }); archive.finalize(); }); } async build(entry: string, output: string) { - await this.prepare(entry); - await this.zip(output); - const data = await this.encode(output); + const code = await this.bundle(entry); + await this.zip(entry, code, output); + const content = await this.encode(output); await this.checkSize(output); await this.cleanup(output); - return data; + return content; } async encode(outputPath: string) { @@ -108,7 +103,6 @@ export class Bundler { } async cleanup(outputPath: string) { - this.moduleMap = new Map(); await unlink(outputPath); } } diff --git a/src/push/index.ts b/src/push/index.ts index ec64b982..b5c872ff 100644 --- a/src/push/index.ts +++ b/src/push/index.ts @@ -24,7 +24,7 @@ */ import { readFile, writeFile } from 'fs/promises'; import { prompt } from 'enquirer'; -import { bold, grey } from 'kleur/colors'; +import { bold, underline } from 'kleur/colors'; import { getLocalMonitors, buildMonitorSchema, @@ -57,8 +57,11 @@ import { isBulkAPISupported, isLightweightMonitorSupported, logDiff, + logGroups, + printBytes, } from './utils'; -import { log } from '../core/logger'; +import { run } from "./run-local" + export async function push(monitors: Monitor[], options: PushOptions) { if (parseInt(process.env.CHUNK_SIZE) > 250) { @@ -84,7 +87,7 @@ export async function push(monitors: Monitor[], options: PushOptions) { const { monitors: remote } = await bulkGetMonitors(options); progress(`preparing ${monitors.length} monitors`); - const schemas = await buildMonitorSchema(monitors, true); + const { schemas, sizes } = await buildMonitorSchema(monitors, true); const local = getLocalMonitors(schemas); const { newIDs, changedIDs, removedIDs, unchangedIDs } = diffMonitorHashIDs( @@ -92,6 +95,18 @@ export async function push(monitors: Monitor[], options: PushOptions) { remote ); logDiff(newIDs, changedIDs, removedIDs, unchangedIDs); + if (options.dryRun) { + logGroups(sizes, newIDs, changedIDs, removedIDs, unchangedIDs); + // show bundle size for the whole project + let totalSize = 0; + for (const value of sizes.values()) { + totalSize += value; + } + progress("total size of the monitors payload is " + printBytes(totalSize)); + await run(schemas); + progress('Dry run completed'); + return; + } const updatedMonitors = new Set([...changedIDs, ...newIDs]); if (updatedMonitors.size > 0) { @@ -106,7 +121,6 @@ export async function push(monitors: Monitor[], options: PushOptions) { } if (removedIDs.size > 0) { - log(`deleting monitor ids: ${Array.from(removedIDs.keys()).join(', ')}`); if (updatedMonitors.size === 0 && unchangedIDs.size === 0) { await confirmDelete( `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`, @@ -126,8 +140,7 @@ export async function push(monitors: Monitor[], options: PushOptions) { ); } } - - done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`); + done(`Pushed: ${underline(getMonitorManagementURL(options.url))} `); } async function confirmDelete(message: string, skip: boolean) { @@ -170,7 +183,7 @@ export function formatDuplicateError(monitors: Set) { let inner = ''; for (const monitor of monitors) { const { config, source } = monitor; - inner += `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`; + inner += `* ${config.id} - ${source.file}:${source.line}:${source.column} \n`; } outer += indent(inner); return outer; @@ -301,7 +314,7 @@ export async function pushLegacy(monitors: Monitor[], options: PushOptions) { let schemas: MonitorSchema[] = []; if (monitors.length > 0) { progress(`preparing ${monitors.length} monitors`); - schemas = await buildMonitorSchema(monitors, false); + ({ schemas } = await buildMonitorSchema(monitors, false)); const chunks = getChunks(schemas, 10); for (const chunk of chunks) { await liveProgress( @@ -320,7 +333,7 @@ export async function pushLegacy(monitors: Monitor[], options: PushOptions) { `deleting all stale monitors` ); - done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`); + done(`Pushed: ${underline(getMonitorManagementURL(options.url))}`); } // prints warning if any of the monitors has throttling settings enabled during push diff --git a/src/push/monitor.ts b/src/push/monitor.ts index 85f39d83..81cb4940 100644 --- a/src/push/monitor.ts +++ b/src/push/monitor.ts @@ -42,6 +42,8 @@ import { isParamOptionSupported, normalizeMonitorName } from './utils'; // Allowed extensions for lightweight monitor files const ALLOWED_LW_EXTENSIONS = ['.yml', '.yaml']; +// 1500kB Max Gzipped limit for bundled monitor code to be pushed as Kibana project monitors. +const SIZE_LIMIT_KB = 1500; export type MonitorSchema = Omit & { locations: string[]; @@ -131,6 +133,7 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) { await mkdir(bundlePath, { recursive: true }); const bundler = new Bundler(); const schemas: MonitorSchema[] = []; + const sizes: Map = new Map(); for (const monitor of monitors) { const { source, config, filter, type } = monitor; @@ -148,6 +151,15 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) { monitor.setContent(content); Object.assign(schema, { content, filter }); } + const size = monitor.size(); + const sizeKB = Math.round(size / 1000); + if (sizeKB > SIZE_LIMIT_KB) { + let outer = bold(`Aborted: Bundled code ${sizeKB}kB exceeds the recommended ${SIZE_LIMIT_KB}kB limit. Please check the dependencies imported.\n`); + const inner = `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`; + outer += indent(inner); + throw red(outer); + } + sizes.set(config.id, size); /** * Generate hash only after the bundled content is created * to capture code changes in imported files @@ -159,7 +171,7 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) { } await rm(bundlePath, { recursive: true }); - return schemas; + return { schemas, sizes }; } export async function createLightweightMonitors( diff --git a/src/push/run-local.ts b/src/push/run-local.ts new file mode 100644 index 00000000..33dd4c06 --- /dev/null +++ b/src/push/run-local.ts @@ -0,0 +1,139 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import { execFileSync, spawn } from "child_process"; +import { MonitorSchema } from "./monitor"; +import { rm, writeFile } from "fs/promises"; +import { createReadStream } from "fs"; +import { Extract } from "unzip-stream" + +async function unzipFile(zipPath, destination) { + return new Promise((resolve, reject) => { + createReadStream(zipPath) + .pipe(Extract({ path: destination })) + .on('close', resolve) + .on('error', (err) => { + reject(new Error(`failed to extract zip ${zipPath} : ${err.message}`)); + }); + }); +} + +async function runNpmInstall(directory) { + return new Promise((resolve, reject) => { + const flags = [ + "--no-audit", // Prevent audit checks + "--no-update-notifier", // Prevent update checks + "--no-fund", // No need for package funding messages here + "--package-lock=false", // no need to write package lock here + "--progress=false", // no need to display progress + ] + + const npmInstall = spawn('npm', ['install', ...flags], + { cwd: directory, stdio: 'ignore' }); + npmInstall.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`npm install failed with exit code ${code}`)); + } + }); + npmInstall.on('error', (err) => { + reject(new Error(`Failed to start npm install: ${err.message}`)); + }); + }); +} + +async function runTest(directory, schema: MonitorSchema) { + return new Promise((resolve, reject) => { + const runTest = spawn('npx', [ + '@elastic/synthetics', + '.', + '--playwright-options', + JSON.stringify(schema.playwrightOptions), + '--params', + JSON.stringify(schema.params), + ], { + cwd: directory, stdio: 'inherit' + }); + + runTest.on('close', (code) => { + if (code === 0) { + resolve(); + } + else { + reject(new Error(`Exited with code ${code}`)); + } + }); + + runTest.on('error', (err) => { + reject(new Error(`Failed to execute @elastic/synthetics : ${err.message}`)); + }); + }) +} + +async function writePkgJSON(dir: string) { + // lookup installed bin path of a node module + const resolvedPath = execFileSync('which', ['elastic-synthetics'], { encoding: 'utf8' }).trim(); + const installedPath = resolvedPath.replace("bin/elastic-synthetics", "lib/node_modules/@elastic/synthetics") + + const packageJsonContent = { + name: "project-journey", + private: "true", + dependencies: { + "@elastic/synthetics": `file:${installedPath}`, + }, + }; + await writeFile(`${dir}/package.json`, JSON.stringify(packageJsonContent, null, 2), "utf-8"); +} + + +async function extractAndRun(schema: MonitorSchema) { + if (schema.type !== "browser") { + return; + } + const content = schema.content; + const fileName = Date.now(); + const zipPath = `/tmp/synthetics-zip-${fileName}.zip`; + const unzipPath = `/tmp/synthetics-unzip-${fileName}`; + try { + await writeFile(zipPath, content, "base64"); + await unzipFile(zipPath, unzipPath); + await writePkgJSON(unzipPath); + await runNpmInstall(unzipPath); + await runTest(unzipPath, schema); + } catch (err) { + // console.error(err); + } finally { + await rm(zipPath, { recursive: true, force: true }); + await rm(unzipPath, { recursive: true, force: true }); + } +} + +export async function run(schemas: MonitorSchema[]) { + for (const schema of schemas) { + await extractAndRun(schema); + } +} + diff --git a/src/push/utils.ts b/src/push/utils.ts index 7f0b70c5..f1a54c13 100644 --- a/src/push/utils.ts +++ b/src/push/utils.ts @@ -25,7 +25,7 @@ import semver from 'semver'; import { progress, removeTrailingSlash } from '../helpers'; -import { green, red, grey, yellow } from 'kleur/colors'; +import { green, red, grey, yellow, Colorize, bold } from 'kleur/colors'; import { PushOptions } from '../common_types'; import { Monitor } from '../dsl/monitor'; @@ -44,6 +44,50 @@ export function logDiff>( ); } +export function logGroups>( + sizes: Map, + newIDs: T, + changedIDs: T, + removedIDs: T, + unchangedIDs: T +) { + console.groupCollapsed(); + logGroup(sizes, 'Added', newIDs, green); + logGroup(sizes, 'Updated', changedIDs, yellow); + logGroup(sizes, 'Removed', removedIDs, red); + logGroup(sizes, 'Unchanged', unchangedIDs, grey); + console.groupEnd() +} + +function logGroup(sizes: Map, name: string, ids: Set, color: Colorize) { + if (ids.size === 0) return; + printLine(process.stdout.columns - 2); + console.groupCollapsed(color(bold(name))); + [...ids].forEach(id => { + console.log(grey(`- ${id} (${printBytes(sizes.get(id))})`)); + }); + console.groupEnd(); +} + +function printLine(length: number = process.stdout.columns) { + console.log(grey("-").repeat(length)); +} + +const BYTE_UNITS = [ + 'B', + 'kB', + 'MB', + 'GB', + 'TB', + 'PB', +]; +export function printBytes(bytes: number) { + if (bytes <= 0) return '0 B'; + const exponent = Math.min(Math.floor(Math.log10(bytes) / 3), BYTE_UNITS.length - 1); + bytes /= 1000 ** exponent; + return `${(bytes.toFixed(1))} ${BYTE_UNITS[exponent]}`; +} + export function getChunks(arr: Array, size: number): Array { const chunks = []; for (let i = 0; i < arr.length; i += size) { From 46ec9291d7449133fd851c7d64df1e10f807462c Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Wed, 18 Dec 2024 18:43:17 -0800 Subject: [PATCH 2/7] move lookup code --- src/push/run-local.ts | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/push/run-local.ts b/src/push/run-local.ts index 33dd4c06..f0c4ddce 100644 --- a/src/push/run-local.ts +++ b/src/push/run-local.ts @@ -78,14 +78,7 @@ async function runTest(directory, schema: MonitorSchema) { cwd: directory, stdio: 'inherit' }); - runTest.on('close', (code) => { - if (code === 0) { - resolve(); - } - else { - reject(new Error(`Exited with code ${code}`)); - } - }); + runTest.on('close', resolve); runTest.on('error', (err) => { reject(new Error(`Failed to execute @elastic/synthetics : ${err.message}`)); @@ -93,23 +86,19 @@ async function runTest(directory, schema: MonitorSchema) { }) } -async function writePkgJSON(dir: string) { - // lookup installed bin path of a node module - const resolvedPath = execFileSync('which', ['elastic-synthetics'], { encoding: 'utf8' }).trim(); - const installedPath = resolvedPath.replace("bin/elastic-synthetics", "lib/node_modules/@elastic/synthetics") - +async function writePkgJSON(dir: string, synthPath: string) { const packageJsonContent = { name: "project-journey", private: "true", dependencies: { - "@elastic/synthetics": `file:${installedPath}`, + "@elastic/synthetics": `file:${synthPath}`, }, }; await writeFile(`${dir}/package.json`, JSON.stringify(packageJsonContent, null, 2), "utf-8"); } -async function extractAndRun(schema: MonitorSchema) { +async function extractAndRun(schema: MonitorSchema, synthPath: string) { if (schema.type !== "browser") { return; } @@ -120,11 +109,11 @@ async function extractAndRun(schema: MonitorSchema) { try { await writeFile(zipPath, content, "base64"); await unzipFile(zipPath, unzipPath); - await writePkgJSON(unzipPath); + await writePkgJSON(unzipPath, synthPath); await runNpmInstall(unzipPath); await runTest(unzipPath, schema); } catch (err) { - // console.error(err); + console.error(err); } finally { await rm(zipPath, { recursive: true, force: true }); await rm(unzipPath, { recursive: true, force: true }); @@ -132,8 +121,12 @@ async function extractAndRun(schema: MonitorSchema) { } export async function run(schemas: MonitorSchema[]) { + // lookup installed bin path of a node module + const resolvedPath = execFileSync('which', ['elastic-synthetics'], { encoding: 'utf8' }).trim(); + const installedPath = resolvedPath.replace("bin/elastic-synthetics", "lib/node_modules/@elastic/synthetics") + for (const schema of schemas) { - await extractAndRun(schema); + await extractAndRun(schema, installedPath); } } From 757e82baa6244623417eae81a3c64e1ec506aa0b Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Thu, 19 Dec 2024 15:37:35 -0800 Subject: [PATCH 3/7] unpack all journeys and run local --- src/core/globals.ts | 12 ++++++++++ src/core/logger.ts | 10 ++------- src/push/index.ts | 11 +++++++--- src/push/run-local.ts | 51 +++++++++++++++++++++---------------------- src/push/utils.ts | 1 + 5 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/core/globals.ts b/src/core/globals.ts index fa615a8f..77f80b6d 100644 --- a/src/core/globals.ts +++ b/src/core/globals.ts @@ -34,4 +34,16 @@ if (!global[SYNTHETICS_RUNNER]) { global[SYNTHETICS_RUNNER] = new Runner(); } +/** + * Set debug based on DEBUG ENV and namespace - synthetics + */ +if (process.env.DEBUG && process.env.DEBUG.includes('synthetics')) { + process.env['__SYNTHETICS__DEBUG__'] = '1'; +} + export const runner: Runner = global[SYNTHETICS_RUNNER]; + +// is Debug mode enabled +export function inDebugMode() { + return !!process.env['__SYNTHETICS__DEBUG__']; +} diff --git a/src/core/logger.ts b/src/core/logger.ts index bb28e4d5..c3713c00 100644 --- a/src/core/logger.ts +++ b/src/core/logger.ts @@ -25,16 +25,10 @@ import { grey, cyan, dim, italic } from 'kleur/colors'; import { now } from '../helpers'; - -/** - * Set debug based on DEBUG ENV and namespace - synthetics - */ -if (process.env.DEBUG && process.env.DEBUG.includes('synthetics')) { - process.env['__SYNTHETICS__DEBUG__'] = '1'; -} +import { inDebugMode } from './globals'; export function log(msg) { - if (!process.env['__SYNTHETICS__DEBUG__'] || !msg) { + if (!inDebugMode() || !msg) { return; } if (typeof msg === 'object') { diff --git a/src/push/index.ts b/src/push/index.ts index b5c872ff..e8b56354 100644 --- a/src/push/index.ts +++ b/src/push/index.ts @@ -60,7 +60,8 @@ import { logGroups, printBytes, } from './utils'; -import { run } from "./run-local" +import { runLocal } from "./run-local" +import { inDebugMode } from '../core/globals'; export async function push(monitors: Monitor[], options: PushOptions) { @@ -95,7 +96,7 @@ export async function push(monitors: Monitor[], options: PushOptions) { remote ); logDiff(newIDs, changedIDs, removedIDs, unchangedIDs); - if (options.dryRun) { + if (inDebugMode()) { logGroups(sizes, newIDs, changedIDs, removedIDs, unchangedIDs); // show bundle size for the whole project let totalSize = 0; @@ -103,7 +104,11 @@ export async function push(monitors: Monitor[], options: PushOptions) { totalSize += value; } progress("total size of the monitors payload is " + printBytes(totalSize)); - await run(schemas); + } + + if (options.dryRun) { + progress('Running browser monitors in dry run mode'); + await runLocal(schemas); progress('Dry run completed'); return; } diff --git a/src/push/run-local.ts b/src/push/run-local.ts index f0c4ddce..250a1958 100644 --- a/src/push/run-local.ts +++ b/src/push/run-local.ts @@ -28,15 +28,16 @@ import { MonitorSchema } from "./monitor"; import { rm, writeFile } from "fs/promises"; import { createReadStream } from "fs"; import { Extract } from "unzip-stream" +import { red } from "kleur/colors"; async function unzipFile(zipPath, destination) { return new Promise((resolve, reject) => { createReadStream(zipPath) .pipe(Extract({ path: destination })) .on('close', resolve) - .on('error', (err) => { - reject(new Error(`failed to extract zip ${zipPath} : ${err.message}`)); - }); + .on('error', (err) => + reject(new Error(`failed to extract zip ${zipPath} : ${err.message}`)) + ); }); } @@ -59,9 +60,9 @@ async function runNpmInstall(directory) { reject(new Error(`npm install failed with exit code ${code}`)); } }); - npmInstall.on('error', (err) => { - reject(new Error(`Failed to start npm install: ${err.message}`)); - }); + npmInstall.on('error', (err) => + reject(new Error(`failed to setup: ${err.message}`)) + ); }); } @@ -79,7 +80,6 @@ async function runTest(directory, schema: MonitorSchema) { }); runTest.on('close', resolve); - runTest.on('error', (err) => { reject(new Error(`Failed to execute @elastic/synthetics : ${err.message}`)); }); @@ -98,35 +98,34 @@ async function writePkgJSON(dir: string, synthPath: string) { } -async function extractAndRun(schema: MonitorSchema, synthPath: string) { +async function extract(schema: MonitorSchema, zipPath: string, unzipPath: string) { if (schema.type !== "browser") { return; } const content = schema.content; - const fileName = Date.now(); - const zipPath = `/tmp/synthetics-zip-${fileName}.zip`; - const unzipPath = `/tmp/synthetics-unzip-${fileName}`; + await writeFile(zipPath, content, "base64"); + await unzipFile(zipPath, unzipPath); +} + +export async function runLocal(schemas: MonitorSchema[]) { + // lookup installed bin path of a node module + const resolvedPath = execFileSync('which', ['elastic-synthetics'], { encoding: 'utf8' }).trim(); + const synthPath = resolvedPath.replace("bin/elastic-synthetics", "lib/node_modules/@elastic/synthetics") + const filename = Date.now(); + const zipPath = `/tmp/synthetics-zip-${filename}.zip`; + const unzipPath = `/tmp/synthetics-unzip-${filename}`; try { - await writeFile(zipPath, content, "base64"); - await unzipFile(zipPath, unzipPath); + for (const schema of schemas) { + await extract(schema, zipPath, unzipPath); + } await writePkgJSON(unzipPath, synthPath); await runNpmInstall(unzipPath); - await runTest(unzipPath, schema); - } catch (err) { - console.error(err); + await runTest(unzipPath, schemas[0]); + } catch (e) { + throw red(`Aborted: ${e.message}`); } finally { await rm(zipPath, { recursive: true, force: true }); await rm(unzipPath, { recursive: true, force: true }); } } -export async function run(schemas: MonitorSchema[]) { - // lookup installed bin path of a node module - const resolvedPath = execFileSync('which', ['elastic-synthetics'], { encoding: 'utf8' }).trim(); - const installedPath = resolvedPath.replace("bin/elastic-synthetics", "lib/node_modules/@elastic/synthetics") - - for (const schema of schemas) { - await extractAndRun(schema, installedPath); - } -} - diff --git a/src/push/utils.ts b/src/push/utils.ts index f1a54c13..8d188ca0 100644 --- a/src/push/utils.ts +++ b/src/push/utils.ts @@ -61,6 +61,7 @@ export function logGroups>( function logGroup(sizes: Map, name: string, ids: Set, color: Colorize) { if (ids.size === 0) return; + // under collapsed group, so giving 2 space for padding printLine(process.stdout.columns - 2); console.groupCollapsed(color(bold(name))); [...ids].forEach(id => { From b2babcc1d6d02ee310b275c2238d7b9315b181a1 Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Thu, 19 Dec 2024 15:41:04 -0800 Subject: [PATCH 4/7] fix tests --- __tests__/push/monitor.test.ts | 14 +++++++------- src/push/index.ts | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/__tests__/push/monitor.test.ts b/__tests__/push/monitor.test.ts index 0d890c96..e982dddd 100644 --- a/__tests__/push/monitor.test.ts +++ b/__tests__/push/monitor.test.ts @@ -70,11 +70,11 @@ describe('Monitors', () => { }); it('build lightweight monitor schema', async () => { - const schema = await buildMonitorSchema( + const { schemas } = await buildMonitorSchema( [createTestMonitor('heartbeat.yml', 'http')], true ); - expect(schema[0]).toEqual({ + expect(schemas[0]).toEqual({ id: 'test-monitor', name: 'test', schedule: 10, @@ -89,8 +89,8 @@ describe('Monitors', () => { it('build browser monitor schema', async () => { const monitor = createTestMonitor('example.journey.ts'); - const schema = await buildMonitorSchema([monitor], true); - expect(schema[0]).toEqual({ + const { schemas } = await buildMonitorSchema([monitor], true); + expect(schemas[0]).toEqual({ id: 'test-monitor', name: 'test', schedule: 10, @@ -106,9 +106,9 @@ describe('Monitors', () => { fields: { area: 'website' }, }); monitor.update({ locations: ['brazil'], fields: { env: 'dev' } }); - const schema1 = await buildMonitorSchema([monitor], true); - expect(schema1[0].hash).not.toEqual(schema[0].hash); - expect(schema1[0].fields).toEqual({ + const { schemas: schemas1 } = await buildMonitorSchema([monitor], true); + expect(schemas1[0].hash).not.toEqual(schemas[0].hash); + expect(schemas1[0].fields).toEqual({ area: 'website', env: 'dev', }); diff --git a/src/push/index.ts b/src/push/index.ts index e8b56354..d6b25553 100644 --- a/src/push/index.ts +++ b/src/push/index.ts @@ -188,7 +188,7 @@ export function formatDuplicateError(monitors: Set) { let inner = ''; for (const monitor of monitors) { const { config, source } = monitor; - inner += `* ${config.id} - ${source.file}:${source.line}:${source.column} \n`; + inner += `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`; } outer += indent(inner); return outer; From b864c76ff06372a627c18a885807da3d3038df52 Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Thu, 19 Dec 2024 16:15:55 -0800 Subject: [PATCH 5/7] some cleanup --- src/push/bundler.ts | 16 +--------------- src/push/run-local.ts | 7 ++++--- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/push/bundler.ts b/src/push/bundler.ts index a3265eff..53b63d5c 100644 --- a/src/push/bundler.ts +++ b/src/push/bundler.ts @@ -24,16 +24,13 @@ */ import path from 'path'; -import { unlink, readFile, stat } from 'fs/promises'; +import { unlink, readFile } from 'fs/promises'; import { createWriteStream } from 'fs'; import * as esbuild from 'esbuild'; import archiver from 'archiver'; import { commonOptions } from '../core/transform'; import { SyntheticsBundlePlugin } from './plugin'; -// 1500kB Max Gzipped limit for bundled monitor code to be pushed as Kibana project monitors. -const SIZE_LIMIT_KB = 1000; - function relativeToCwd(entry: string) { return path.relative(process.cwd(), entry); } @@ -83,7 +80,6 @@ export class Bundler { const code = await this.bundle(entry); await this.zip(entry, code, output); const content = await this.encode(output); - await this.checkSize(output); await this.cleanup(output); return content; } @@ -92,16 +88,6 @@ export class Bundler { return await readFile(outputPath, 'base64'); } - async checkSize(outputPath: string) { - const { size } = await stat(outputPath); - const sizeKb = size / 1024; - if (sizeKb > SIZE_LIMIT_KB) { - throw new Error( - `Bundled monitor code exceeds the recommended ${SIZE_LIMIT_KB}KB limit. Please check your dependencies and try again.` - ); - } - } - async cleanup(outputPath: string) { await unlink(outputPath); } diff --git a/src/push/run-local.ts b/src/push/run-local.ts index 250a1958..511c9e02 100644 --- a/src/push/run-local.ts +++ b/src/push/run-local.ts @@ -111,15 +111,16 @@ export async function runLocal(schemas: MonitorSchema[]) { // lookup installed bin path of a node module const resolvedPath = execFileSync('which', ['elastic-synthetics'], { encoding: 'utf8' }).trim(); const synthPath = resolvedPath.replace("bin/elastic-synthetics", "lib/node_modules/@elastic/synthetics") - const filename = Date.now(); - const zipPath = `/tmp/synthetics-zip-${filename}.zip`; - const unzipPath = `/tmp/synthetics-unzip-${filename}`; + const rand = Date.now(); + const zipPath = `/tmp/synthetics-zip-${rand}.zip`; + const unzipPath = `/tmp/synthetics-unzip-${rand}`; try { for (const schema of schemas) { await extract(schema, zipPath, unzipPath); } await writePkgJSON(unzipPath, synthPath); await runNpmInstall(unzipPath); + // TODO: figure out a way to collect all params and Playwright options await runTest(unzipPath, schemas[0]); } catch (e) { throw red(`Aborted: ${e.message}`); From c68b561cde19fa6afa1eee0e4c2a00e5470e0b20 Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Fri, 17 Jan 2025 10:15:40 -0800 Subject: [PATCH 6/7] review comments --- src/push/run-local.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/push/run-local.ts b/src/push/run-local.ts index 511c9e02..566f2449 100644 --- a/src/push/run-local.ts +++ b/src/push/run-local.ts @@ -24,11 +24,14 @@ */ import { execFileSync, spawn } from "child_process"; -import { MonitorSchema } from "./monitor"; import { rm, writeFile } from "fs/promises"; import { createReadStream } from "fs"; +import { tmpdir } from "os" import { Extract } from "unzip-stream" import { red } from "kleur/colors"; +import { join } from "path"; +import { pathToFileURL } from "url"; +import { MonitorSchema } from "./monitor"; async function unzipFile(zipPath, destination) { return new Promise((resolve, reject) => { @@ -91,10 +94,10 @@ async function writePkgJSON(dir: string, synthPath: string) { name: "project-journey", private: "true", dependencies: { - "@elastic/synthetics": `file:${synthPath}`, + "@elastic/synthetics": pathToFileURL(synthPath), }, }; - await writeFile(`${dir}/package.json`, JSON.stringify(packageJsonContent, null, 2), "utf-8"); + await writeFile(join(dir, 'package.json'), JSON.stringify(packageJsonContent, null, 2), "utf-8"); } @@ -110,10 +113,12 @@ async function extract(schema: MonitorSchema, zipPath: string, unzipPath: string export async function runLocal(schemas: MonitorSchema[]) { // lookup installed bin path of a node module const resolvedPath = execFileSync('which', ['elastic-synthetics'], { encoding: 'utf8' }).trim(); - const synthPath = resolvedPath.replace("bin/elastic-synthetics", "lib/node_modules/@elastic/synthetics") + const synthPath = resolvedPath.replace( + join('bin', 'elastic-synthetics'), + join('lib', 'node_modules', '@elastic/synthetics')); const rand = Date.now(); - const zipPath = `/tmp/synthetics-zip-${rand}.zip`; - const unzipPath = `/tmp/synthetics-unzip-${rand}`; + const zipPath = join(tmpdir(), `synthetics-zip-${rand}.zip`); + const unzipPath = join(tmpdir(), `synthetics-unzip-${rand}`); try { for (const schema of schemas) { await extract(schema, zipPath, unzipPath); From e9e579a59e69c743564f86836a3fb2bde2fc3823 Mon Sep 17 00:00:00 2001 From: vigneshshanmugam Date: Fri, 17 Jan 2025 10:17:28 -0800 Subject: [PATCH 7/7] fix formatting issues --- src/dsl/monitor.ts | 4 +- src/push/bundler.ts | 2 +- src/push/index.ts | 5 +- src/push/monitor.ts | 4 +- src/push/run-local.ts | 106 ++++++++++++++++++++++++------------------ src/push/utils.ts | 27 +++++------ 6 files changed, 84 insertions(+), 64 deletions(-) diff --git a/src/dsl/monitor.ts b/src/dsl/monitor.ts index 97470994..87d56289 100644 --- a/src/dsl/monitor.ts +++ b/src/dsl/monitor.ts @@ -88,7 +88,7 @@ export class Monitor { content?: string; source?: Location; filter: MonitorFilter; - constructor(public config: MonitorConfig = {}) { } + constructor(public config: MonitorConfig = {}) {} /** * Treat the creation time config with `monitor.use` as source of truth by * merging the values coming from CLI and Synthetics config file @@ -162,7 +162,7 @@ export class Monitor { * Returns the size of the monitor in bytes which is sent as payload to Kibana */ size() { - return JSON.stringify(this).length + return JSON.stringify(this).length; } validate() { diff --git a/src/push/bundler.ts b/src/push/bundler.ts index 53b63d5c..6f009d96 100644 --- a/src/push/bundler.ts +++ b/src/push/bundler.ts @@ -53,7 +53,7 @@ export class Bundler { if (result.errors.length > 0) { throw result.errors; } - return result.outputFiles[0].text + return result.outputFiles[0].text; } async zip(source: string, code: string, dest: string) { diff --git a/src/push/index.ts b/src/push/index.ts index 9541411d..25ea0caa 100644 --- a/src/push/index.ts +++ b/src/push/index.ts @@ -60,10 +60,9 @@ import { logGroups, printBytes, } from './utils'; -import { runLocal } from "./run-local" +import { runLocal } from './run-local'; import { inDebugMode } from '../core/globals'; - export async function push(monitors: Monitor[], options: PushOptions) { if (parseInt(process.env.CHUNK_SIZE) > 250) { throw error( @@ -103,7 +102,7 @@ export async function push(monitors: Monitor[], options: PushOptions) { for (const value of sizes.values()) { totalSize += value; } - progress("total size of the monitors payload is " + printBytes(totalSize)); + progress('total size of the monitors payload is ' + printBytes(totalSize)); } if (options.dryRun) { diff --git a/src/push/monitor.ts b/src/push/monitor.ts index e5d86574..421d2261 100644 --- a/src/push/monitor.ts +++ b/src/push/monitor.ts @@ -154,7 +154,9 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) { const size = monitor.size(); const sizeKB = Math.round(size / 1000); if (sizeKB > SIZE_LIMIT_KB) { - let outer = bold(`Aborted: Bundled code ${sizeKB}kB exceeds the recommended ${SIZE_LIMIT_KB}kB limit. Please check the dependencies imported.\n`); + let outer = bold( + `Aborted: Bundled code ${sizeKB}kB exceeds the recommended ${SIZE_LIMIT_KB}kB limit. Please check the dependencies imported.\n` + ); const inner = `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`; outer += indent(inner); throw red(outer); diff --git a/src/push/run-local.ts b/src/push/run-local.ts index 566f2449..7b082c73 100644 --- a/src/push/run-local.ts +++ b/src/push/run-local.ts @@ -23,22 +23,22 @@ * */ -import { execFileSync, spawn } from "child_process"; -import { rm, writeFile } from "fs/promises"; -import { createReadStream } from "fs"; -import { tmpdir } from "os" -import { Extract } from "unzip-stream" -import { red } from "kleur/colors"; -import { join } from "path"; -import { pathToFileURL } from "url"; -import { MonitorSchema } from "./monitor"; +import { execFileSync, spawn } from 'child_process'; +import { rm, writeFile } from 'fs/promises'; +import { createReadStream } from 'fs'; +import { tmpdir } from 'os'; +import { Extract } from 'unzip-stream'; +import { red } from 'kleur/colors'; +import { join } from 'path'; +import { pathToFileURL } from 'url'; +import { MonitorSchema } from './monitor'; async function unzipFile(zipPath, destination) { return new Promise((resolve, reject) => { createReadStream(zipPath) .pipe(Extract({ path: destination })) .on('close', resolve) - .on('error', (err) => + .on('error', err => reject(new Error(`failed to extract zip ${zipPath} : ${err.message}`)) ); }); @@ -47,23 +47,25 @@ async function unzipFile(zipPath, destination) { async function runNpmInstall(directory) { return new Promise((resolve, reject) => { const flags = [ - "--no-audit", // Prevent audit checks - "--no-update-notifier", // Prevent update checks - "--no-fund", // No need for package funding messages here - "--package-lock=false", // no need to write package lock here - "--progress=false", // no need to display progress - ] + '--no-audit', // Prevent audit checks + '--no-update-notifier', // Prevent update checks + '--no-fund', // No need for package funding messages here + '--package-lock=false', // no need to write package lock here + '--progress=false', // no need to display progress + ]; - const npmInstall = spawn('npm', ['install', ...flags], - { cwd: directory, stdio: 'ignore' }); - npmInstall.on('close', (code) => { + const npmInstall = spawn('npm', ['install', ...flags], { + cwd: directory, + stdio: 'ignore', + }); + npmInstall.on('close', code => { if (code === 0) { resolve(); } else { reject(new Error(`npm install failed with exit code ${code}`)); } }); - npmInstall.on('error', (err) => + npmInstall.on('error', err => reject(new Error(`failed to setup: ${err.message}`)) ); }); @@ -71,51 +73,68 @@ async function runNpmInstall(directory) { async function runTest(directory, schema: MonitorSchema) { return new Promise((resolve, reject) => { - const runTest = spawn('npx', [ - '@elastic/synthetics', - '.', - '--playwright-options', - JSON.stringify(schema.playwrightOptions), - '--params', - JSON.stringify(schema.params), - ], { - cwd: directory, stdio: 'inherit' - }); + const runTest = spawn( + 'npx', + [ + '@elastic/synthetics', + '.', + '--playwright-options', + JSON.stringify(schema.playwrightOptions), + '--params', + JSON.stringify(schema.params), + ], + { + cwd: directory, + stdio: 'inherit', + } + ); runTest.on('close', resolve); - runTest.on('error', (err) => { - reject(new Error(`Failed to execute @elastic/synthetics : ${err.message}`)); + runTest.on('error', err => { + reject( + new Error(`Failed to execute @elastic/synthetics : ${err.message}`) + ); }); - }) + }); } async function writePkgJSON(dir: string, synthPath: string) { const packageJsonContent = { - name: "project-journey", - private: "true", + name: 'project-journey', + private: 'true', dependencies: { - "@elastic/synthetics": pathToFileURL(synthPath), + '@elastic/synthetics': pathToFileURL(synthPath), }, }; - await writeFile(join(dir, 'package.json'), JSON.stringify(packageJsonContent, null, 2), "utf-8"); + await writeFile( + join(dir, 'package.json'), + JSON.stringify(packageJsonContent, null, 2), + 'utf-8' + ); } - -async function extract(schema: MonitorSchema, zipPath: string, unzipPath: string) { - if (schema.type !== "browser") { +async function extract( + schema: MonitorSchema, + zipPath: string, + unzipPath: string +) { + if (schema.type !== 'browser') { return; } const content = schema.content; - await writeFile(zipPath, content, "base64"); + await writeFile(zipPath, content, 'base64'); await unzipFile(zipPath, unzipPath); } export async function runLocal(schemas: MonitorSchema[]) { // lookup installed bin path of a node module - const resolvedPath = execFileSync('which', ['elastic-synthetics'], { encoding: 'utf8' }).trim(); + const resolvedPath = execFileSync('which', ['elastic-synthetics'], { + encoding: 'utf8', + }).trim(); const synthPath = resolvedPath.replace( join('bin', 'elastic-synthetics'), - join('lib', 'node_modules', '@elastic/synthetics')); + join('lib', 'node_modules', '@elastic/synthetics') + ); const rand = Date.now(); const zipPath = join(tmpdir(), `synthetics-zip-${rand}.zip`); const unzipPath = join(tmpdir(), `synthetics-unzip-${rand}`); @@ -134,4 +153,3 @@ export async function runLocal(schemas: MonitorSchema[]) { await rm(unzipPath, { recursive: true, force: true }); } } - diff --git a/src/push/utils.ts b/src/push/utils.ts index 0e8b6803..23008ce3 100644 --- a/src/push/utils.ts +++ b/src/push/utils.ts @@ -56,10 +56,15 @@ export function logGroups>( logGroup(sizes, 'Updated', changedIDs, yellow); logGroup(sizes, 'Removed', removedIDs, red); logGroup(sizes, 'Unchanged', unchangedIDs, grey); - console.groupEnd() + console.groupEnd(); } -function logGroup(sizes: Map, name: string, ids: Set, color: Colorize) { +function logGroup( + sizes: Map, + name: string, + ids: Set, + color: Colorize +) { if (ids.size === 0) return; // under collapsed group, so giving 2 space for padding printLine(process.stdout.columns - 2); @@ -71,22 +76,18 @@ function logGroup(sizes: Map, name: string, ids: Set, co } function printLine(length: number = process.stdout.columns) { - console.log(grey("-").repeat(length)); + console.log(grey('-').repeat(length)); } -const BYTE_UNITS = [ - 'B', - 'kB', - 'MB', - 'GB', - 'TB', - 'PB', -]; +const BYTE_UNITS = ['B', 'kB', 'MB', 'GB', 'TB', 'PB']; export function printBytes(bytes: number) { if (bytes <= 0) return '0 B'; - const exponent = Math.min(Math.floor(Math.log10(bytes) / 3), BYTE_UNITS.length - 1); + const exponent = Math.min( + Math.floor(Math.log10(bytes) / 3), + BYTE_UNITS.length - 1 + ); bytes /= 1000 ** exponent; - return `${(bytes.toFixed(1))} ${BYTE_UNITS[exponent]}`; + return `${bytes.toFixed(1)} ${BYTE_UNITS[exponent]}`; } export function getChunks(arr: Array, size: number): Array {