From 11f1562c5ea2d3fff94f8a48673c578ebd70be4b Mon Sep 17 00:00:00 2001 From: Young Min Kin Date: Wed, 9 Sep 2020 06:38:56 +0900 Subject: [PATCH 1/4] refactor(npm): extract updateYarnOffline function --- lib/manager/npm/post-update/index.ts | 82 +++++++++++++++------------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/lib/manager/npm/post-update/index.ts b/lib/manager/npm/post-update/index.ts index 0a8b3e1de0ca03..5f2eb846f329ad 100644 --- a/lib/manager/npm/post-update/index.ts +++ b/lib/manager/npm/post-update/index.ts @@ -336,6 +336,50 @@ async function resetNpmrcContent( } } +// istanbul ignore next +async function updateYarnOffline( + lockFileDir: string, + localDir: string, + updatedArtifacts: UpdatedArtifcats[] +): Promise { + try { + const yarnrc = await getFile(upath.join(lockFileDir, '.yarnrc')); + if (yarnrc) { + const mirrorLine = yarnrc + .split('\n') + .find((line) => line.startsWith('yarn-offline-mirror ')); + if (mirrorLine) { + const mirrorPath = mirrorLine + .split(' ')[1] + .replace(/"/g, '') + .replace(/\/?$/, '/'); + const resolvedPath = upath.join(lockFileDir, mirrorPath); + logger.debug('Found yarn offline mirror: ' + resolvedPath); + const status = await getRepoStatus(); + for (const f of status.modified.concat(status.not_added)) { + if (f.startsWith(resolvedPath)) { + const localModified = upath.join(localDir, f); + updatedArtifacts.push({ + name: f, + contents: await readFile(localModified), + }); + } + } + for (const f of status.deleted || []) { + if (f.startsWith(resolvedPath)) { + updatedArtifacts.push({ + name: '|delete|', + contents: f, + }); + } + } + } + } + } catch (err) { + logger.error({ err }, 'Error updating yarn offline packages'); + } +} + export interface WriteExistingFilesResult { artifactErrors: ArtifactError[]; updatedArtifacts: UpdatedArtifcats[]; @@ -538,43 +582,7 @@ export async function getAdditionalFiles( name: lockFileName, contents: res.lockFile, }); - // istanbul ignore next - try { - const yarnrc = await getFile(upath.join(lockFileDir, '.yarnrc')); - if (yarnrc) { - const mirrorLine = yarnrc - .split('\n') - .find((line) => line.startsWith('yarn-offline-mirror ')); - if (mirrorLine) { - const mirrorPath = mirrorLine - .split(' ')[1] - .replace(/"/g, '') - .replace(/\/?$/, '/'); - const resolvedPath = upath.join(lockFileDir, mirrorPath); - logger.debug('Found yarn offline mirror: ' + resolvedPath); - const status = await getRepoStatus(); - for (const f of status.modified.concat(status.not_added)) { - if (f.startsWith(resolvedPath)) { - const localModified = upath.join(config.localDir, f); - updatedArtifacts.push({ - name: f, - contents: await readFile(localModified), - }); - } - } - for (const f of status.deleted || []) { - if (f.startsWith(resolvedPath)) { - updatedArtifacts.push({ - name: '|delete|', - contents: f, - }); - } - } - } - } - } catch (err) { - logger.error({ err }, 'Error updating yarn offline packages'); - } + await updateYarnOffline(lockFileDir, config.localDir, updatedArtifacts); } else { logger.debug("yarn.lock hasn't changed"); } From 06ac7aa7c2a21b58097ce6b7d467efe0f56eca95 Mon Sep 17 00:00:00 2001 From: Young Min Kin Date: Wed, 9 Sep 2020 07:22:54 +0900 Subject: [PATCH 2/4] feat(npm): support Yarn 2 offline cache and zero-installs --- lib/manager/npm/post-update/index.ts | 55 +++++++++++++++++++--------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/lib/manager/npm/post-update/index.ts b/lib/manager/npm/post-update/index.ts index 5f2eb846f329ad..76f907a2159cf7 100644 --- a/lib/manager/npm/post-update/index.ts +++ b/lib/manager/npm/post-update/index.ts @@ -1,5 +1,6 @@ import path from 'path'; import is from '@sindresorhus/is'; +import { parseSyml } from '@yarnpkg/parsers'; import upath from 'upath'; import { SYSTEM_INSUFFICIENT_DISK_SPACE } from '../../../constants/error-messages'; import { id as npmId } from '../../../datasource/npm'; @@ -343,8 +344,12 @@ async function updateYarnOffline( updatedArtifacts: UpdatedArtifcats[] ): Promise { try { + const resolvedPaths: string[] = []; const yarnrc = await getFile(upath.join(lockFileDir, '.yarnrc')); + const yarnrcYml = await getFile(upath.join(lockFileDir, '.yarnrc.yml')); + if (yarnrc) { + // Yarn 1 (offline mirror) const mirrorLine = yarnrc .split('\n') .find((line) => line.startsWith('yarn-offline-mirror ')); @@ -353,25 +358,39 @@ async function updateYarnOffline( .split(' ')[1] .replace(/"/g, '') .replace(/\/?$/, '/'); - const resolvedPath = upath.join(lockFileDir, mirrorPath); - logger.debug('Found yarn offline mirror: ' + resolvedPath); - const status = await getRepoStatus(); - for (const f of status.modified.concat(status.not_added)) { - if (f.startsWith(resolvedPath)) { - const localModified = upath.join(localDir, f); - updatedArtifacts.push({ - name: f, - contents: await readFile(localModified), - }); - } + resolvedPaths.push(upath.join(lockFileDir, mirrorPath)); + } + } else if (yarnrcYml) { + // Yarn 2 (offline cache and zero-installs) + const config = parseSyml(yarnrcYml); + resolvedPaths.push( + upath.join(lockFileDir, config.cacheFolder || './.yarn/cache') + ); + + resolvedPaths.push(upath.join(lockFileDir, '.pnp')); + if (config.pnpDataPath) { + resolvedPaths.push(upath.join(lockFileDir, config.pnpDataPath)); + } + } + logger.debug({ resolvedPaths }, 'updateYarnOffline resolvedPaths'); + + if (resolvedPaths.length) { + const status = await getRepoStatus(); + for (const f of status.modified.concat(status.not_added)) { + if (resolvedPaths.some((p) => f.startsWith(p))) { + const localModified = upath.join(localDir, f); + updatedArtifacts.push({ + name: f, + contents: await readFile(localModified), + }); } - for (const f of status.deleted || []) { - if (f.startsWith(resolvedPath)) { - updatedArtifacts.push({ - name: '|delete|', - contents: f, - }); - } + } + for (const f of status.deleted || []) { + if (resolvedPaths.some((p) => f.startsWith(p))) { + updatedArtifacts.push({ + name: '|delete|', + contents: f, + }); } } } From de99046bd51723b76368e97c627f9a4eb46d481a Mon Sep 17 00:00:00 2001 From: Young Min Kin Date: Wed, 9 Sep 2020 09:46:36 +0900 Subject: [PATCH 3/4] refactor: fix typo UpdatedArtifcats --- lib/manager/npm/post-update/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/manager/npm/post-update/index.ts b/lib/manager/npm/post-update/index.ts index 76f907a2159cf7..28cff1704d3882 100644 --- a/lib/manager/npm/post-update/index.ts +++ b/lib/manager/npm/post-update/index.ts @@ -275,7 +275,7 @@ interface ArtifactError { stderr: string; } -interface UpdatedArtifcats { +interface UpdatedArtifacts { name: string; contents: string | Buffer; } @@ -341,7 +341,7 @@ async function resetNpmrcContent( async function updateYarnOffline( lockFileDir: string, localDir: string, - updatedArtifacts: UpdatedArtifcats[] + updatedArtifacts: UpdatedArtifacts[] ): Promise { try { const resolvedPaths: string[] = []; @@ -401,7 +401,7 @@ async function updateYarnOffline( export interface WriteExistingFilesResult { artifactErrors: ArtifactError[]; - updatedArtifacts: UpdatedArtifcats[]; + updatedArtifacts: UpdatedArtifacts[]; } // istanbul ignore next export async function getAdditionalFiles( @@ -410,7 +410,7 @@ export async function getAdditionalFiles( ): Promise { logger.trace({ config }, 'getAdditionalFiles'); const artifactErrors: ArtifactError[] = []; - const updatedArtifacts: UpdatedArtifcats[] = []; + const updatedArtifacts: UpdatedArtifacts[] = []; if (!packageFiles.npm?.length) { return { artifactErrors, updatedArtifacts }; } From 06342b6890c66a4447aa57dcc3d025b5bde1d70d Mon Sep 17 00:00:00 2001 From: Young Min Kin Date: Wed, 9 Sep 2020 17:08:48 +0900 Subject: [PATCH 4/4] fix: check for .yarnrc.yml first before .yarnrc --- lib/manager/npm/post-update/index.ts | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/manager/npm/post-update/index.ts b/lib/manager/npm/post-update/index.ts index 28cff1704d3882..0e034669be70ad 100644 --- a/lib/manager/npm/post-update/index.ts +++ b/lib/manager/npm/post-update/index.ts @@ -345,10 +345,23 @@ async function updateYarnOffline( ): Promise { try { const resolvedPaths: string[] = []; - const yarnrc = await getFile(upath.join(lockFileDir, '.yarnrc')); const yarnrcYml = await getFile(upath.join(lockFileDir, '.yarnrc.yml')); + const yarnrc = await getFile(upath.join(lockFileDir, '.yarnrc')); - if (yarnrc) { + // As .yarnrc.yml overrides .yarnrc in Yarn 1 (https://git.io/JUcco) + // both files may exist, so check for .yarnrc.yml first + if (yarnrcYml) { + // Yarn 2 (offline cache and zero-installs) + const config = parseSyml(yarnrcYml); + resolvedPaths.push( + upath.join(lockFileDir, config.cacheFolder || './.yarn/cache') + ); + + resolvedPaths.push(upath.join(lockFileDir, '.pnp')); + if (config.pnpDataPath) { + resolvedPaths.push(upath.join(lockFileDir, config.pnpDataPath)); + } + } else if (yarnrc) { // Yarn 1 (offline mirror) const mirrorLine = yarnrc .split('\n') @@ -360,17 +373,6 @@ async function updateYarnOffline( .replace(/\/?$/, '/'); resolvedPaths.push(upath.join(lockFileDir, mirrorPath)); } - } else if (yarnrcYml) { - // Yarn 2 (offline cache and zero-installs) - const config = parseSyml(yarnrcYml); - resolvedPaths.push( - upath.join(lockFileDir, config.cacheFolder || './.yarn/cache') - ); - - resolvedPaths.push(upath.join(lockFileDir, '.pnp')); - if (config.pnpDataPath) { - resolvedPaths.push(upath.join(lockFileDir, config.pnpDataPath)); - } } logger.debug({ resolvedPaths }, 'updateYarnOffline resolvedPaths');