From 3ae2c0a3c79a0b8bcbe04e941133aee1a85d3f39 Mon Sep 17 00:00:00 2001 From: secustor Date: Thu, 11 Jul 2024 23:29:11 +0200 Subject: [PATCH 1/2] refactor(manager/circleci): replace regex system with YAML parsing --- lib/config/migration.ts | 2 +- lib/modules/datasource/npm/npmrc.ts | 2 +- .../manager/circleci/__fixtures__/config2.yml | 2 +- .../manager/circleci/__fixtures__/config3.yml | 6 +- .../__snapshots__/extract.spec.ts.snap | 4 - lib/modules/manager/circleci/extract.spec.ts | 1 + lib/modules/manager/circleci/extract.ts | 96 +++++++------------ lib/modules/manager/circleci/schema.ts | 15 +++ lib/modules/manager/gradle/parser/handlers.ts | 2 +- lib/modules/manager/npm/post-update/npm.ts | 2 +- lib/modules/manager/nuget/artifacts.ts | 2 +- 11 files changed, 57 insertions(+), 77 deletions(-) create mode 100644 lib/modules/manager/circleci/schema.ts diff --git a/lib/config/migration.ts b/lib/config/migration.ts index 67f7d7a9035233..32317d4a41e81d 100644 --- a/lib/config/migration.ts +++ b/lib/config/migration.ts @@ -70,7 +70,7 @@ export function migrateConfig(config: RenovateConfig): MigratedConfig { } else if (is.array(val)) { if (is.array(migratedConfig?.[key])) { const newArray = []; - for (const item of migratedConfig[key] as unknown[]) { + for (const item of migratedConfig[key]) { if (is.object(item) && !is.array(item)) { const arrMigrate = migrateConfig(item as RenovateConfig); newArray.push(arrMigrate.migratedConfig); diff --git a/lib/modules/datasource/npm/npmrc.ts b/lib/modules/datasource/npm/npmrc.ts index 73b2e1eef52896..3e44ce476c371d 100644 --- a/lib/modules/datasource/npm/npmrc.ts +++ b/lib/modules/datasource/npm/npmrc.ts @@ -29,7 +29,7 @@ function envReplace(value: any, env = process.env): any { logger.warn('Failed to replace env in config: ' + match); throw new Error('env-replace'); } - return env[envVarName]!; + return env[envVarName]; }); } diff --git a/lib/modules/manager/circleci/__fixtures__/config2.yml b/lib/modules/manager/circleci/__fixtures__/config2.yml index 3f885f3e5c0a15..df2a528d8a2228 100644 --- a/lib/modules/manager/circleci/__fixtures__/config2.yml +++ b/lib/modules/manager/circleci/__fixtures__/config2.yml @@ -9,7 +9,7 @@ orbs: no-version: abc/def # Comments help me understand my work. - volatile: zzz/zzz@volatile # Comments help me understand my work. + volatile: "zzz/zzz@volatile" # Comments help me understand my work. test_plan: &test_plan steps: diff --git a/lib/modules/manager/circleci/__fixtures__/config3.yml b/lib/modules/manager/circleci/__fixtures__/config3.yml index 307ade4d7fcf14..5523b2c39ae3ae 100644 --- a/lib/modules/manager/circleci/__fixtures__/config3.yml +++ b/lib/modules/manager/circleci/__fixtures__/config3.yml @@ -1,18 +1,16 @@ -aliases: +aliases: - &nodejs image: cimg/node:14.8.0 version: 2 jobs: - checkout: - <<: *defaults + checkout: docker: - *nodejs steps: - run: yarn build:runtime release_docker: - <<: *defaults machine: image: ubuntu-1604:201903-01 docker_layer_caching: true diff --git a/lib/modules/manager/circleci/__snapshots__/extract.spec.ts.snap b/lib/modules/manager/circleci/__snapshots__/extract.spec.ts.snap index 77e4c1db6d89f8..99741c70d49791 100644 --- a/lib/modules/manager/circleci/__snapshots__/extract.spec.ts.snap +++ b/lib/modules/manager/circleci/__snapshots__/extract.spec.ts.snap @@ -10,7 +10,6 @@ exports[`modules/manager/circleci/extract extractPackageFile() extracts multiple "depName": "node", "depType": "docker", "replaceString": "node", - "versioning": "docker", }, { "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", @@ -20,7 +19,6 @@ exports[`modules/manager/circleci/extract extractPackageFile() extracts multiple "depName": "node", "depType": "docker", "replaceString": "node:4", - "versioning": "docker", }, { "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", @@ -30,7 +28,6 @@ exports[`modules/manager/circleci/extract extractPackageFile() extracts multiple "depName": "node", "depType": "docker", "replaceString": "node:6", - "versioning": "docker", }, { "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", @@ -40,7 +37,6 @@ exports[`modules/manager/circleci/extract extractPackageFile() extracts multiple "depName": "node", "depType": "docker", "replaceString": "node:8.9.0", - "versioning": "docker", }, ] `; diff --git a/lib/modules/manager/circleci/extract.spec.ts b/lib/modules/manager/circleci/extract.spec.ts index 4222a22a5423ec..827f791248470e 100644 --- a/lib/modules/manager/circleci/extract.spec.ts +++ b/lib/modules/manager/circleci/extract.spec.ts @@ -51,6 +51,7 @@ describe('modules/manager/circleci/extract', () => { const res = extractPackageFile(file3); expect(res?.deps).toMatchObject([ { currentValue: '14.8.0', depName: 'cimg/node' }, + { currentValue: '14.8.0', depName: 'cimg/node' }, ]); }); diff --git a/lib/modules/manager/circleci/extract.ts b/lib/modules/manager/circleci/extract.ts index 9d971de87724c0..4de5c6b58e0ea1 100644 --- a/lib/modules/manager/circleci/extract.ts +++ b/lib/modules/manager/circleci/extract.ts @@ -1,9 +1,11 @@ import { logger } from '../../../logger'; -import { newlineRegex, regEx } from '../../../util/regex'; +import { coerceArray } from '../../../util/array'; +import { parseSingleYaml } from '../../../util/yaml'; import { OrbDatasource } from '../../datasource/orb'; import * as npmVersioning from '../../versioning/npm'; import { getDep } from '../dockerfile/extract'; import type { PackageDependency, PackageFileContent } from '../types'; +import { CircleCiFile } from './schema'; export function extractPackageFile( content: string, @@ -11,70 +13,38 @@ export function extractPackageFile( ): PackageFileContent | null { const deps: PackageDependency[] = []; try { - const lines = content.split(newlineRegex); - for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) { - const line = lines[lineNumber]; - const orbs = regEx(/^\s*orbs:\s*$/).exec(line); - if (orbs) { - logger.trace(`Matched orbs on line ${lineNumber}`); - let foundOrbOrNoop: boolean; - do { - foundOrbOrNoop = false; - const orbLine = lines[lineNumber + 1]; - logger.trace(`orbLine: "${orbLine}"`); - const yamlNoop = regEx(/^\s*(#|$)/).exec(orbLine); - if (yamlNoop) { - logger.debug('orbNoop'); - foundOrbOrNoop = true; - lineNumber += 1; - continue; - } - const orbMatch = regEx(/^\s+([^:]+):\s(.+?)(?:\s*#.*)?$/).exec( - orbLine, - ); - if (orbMatch) { - logger.trace('orbMatch'); - foundOrbOrNoop = true; - lineNumber += 1; - const depName = orbMatch[1]; - const [orbName, currentValue] = orbMatch[2].split('@'); - const dep: PackageDependency = { - depType: 'orb', - depName, - currentValue, - datasource: OrbDatasource.id, - packageName: orbName, - commitMessageTopic: '{{{depName}}} orb', - versioning: npmVersioning.id, - }; - deps.push(dep); - } - } while (foundOrbOrNoop); - } - const match = regEx(/^\s*-? image:\s*'?"?([^\s'"]+)'?"?\s*$/).exec(line); - if (match) { - const currentFrom = match[1]; - const dep = getDep(currentFrom); - logger.debug( - { - depName: dep.depName, - currentValue: dep.currentValue, - currentDigest: dep.currentDigest, - }, - 'CircleCI docker image', - ); - dep.depType = 'docker'; - dep.versioning = 'docker'; - if ( - !dep.depName?.startsWith('ubuntu-') && - !dep.depName?.startsWith('windows-server-') && - !dep.depName?.startsWith('android-') && - dep.depName !== 'android' - ) { - deps.push(dep); - } + const parsed = parseSingleYaml(content, { + customSchema: CircleCiFile, + }); + + for (const [key, orb] of Object.entries(parsed.orbs ?? {})) { + const [packageName, currentValue] = orb.split('@'); + + deps.push({ + depName: key, + packageName, + depType: 'orb', + currentValue, + versioning: npmVersioning.id, + datasource: OrbDatasource.id, + }); + } + + for (const job of Object.values(parsed.jobs)) { + for (const dockerElement of coerceArray(job.docker)) { + deps.push({ + ...getDep(dockerElement.image), + depType: 'docker', + }); } } + + for (const alias of coerceArray(parsed.aliases)) { + deps.push({ + ...getDep(alias.image), + depType: 'docker', + }); + } } catch (err) /* istanbul ignore next */ { logger.debug({ err, packageFile }, 'Error extracting circleci images'); } diff --git a/lib/modules/manager/circleci/schema.ts b/lib/modules/manager/circleci/schema.ts new file mode 100644 index 00000000000000..09c945bd117ee4 --- /dev/null +++ b/lib/modules/manager/circleci/schema.ts @@ -0,0 +1,15 @@ +import { z } from 'zod'; + +export const CircleCiDocker = z.object({ + image: z.string(), +}); + +export const CircleCiJob = z.object({ + docker: z.array(CircleCiDocker).optional(), +}); + +export const CircleCiFile = z.object({ + aliases: z.array(CircleCiDocker).optional(), + jobs: z.record(z.string(), CircleCiJob), + orbs: z.record(z.string()).optional(), +}); diff --git a/lib/modules/manager/gradle/parser/handlers.ts b/lib/modules/manager/gradle/parser/handlers.ts index be12911f0eed9d..2d199f51eb3840 100644 --- a/lib/modules/manager/gradle/parser/handlers.ts +++ b/lib/modules/manager/gradle/parser/handlers.ts @@ -371,7 +371,7 @@ export function handleApplyFrom(ctx: Ctx): Ctx { const matchResult = parseGradle( // TODO #22198 - ctx.fileContents[scriptFilePath]!, + ctx.fileContents[scriptFilePath], ctx.globalVars, scriptFilePath, ctx.fileContents, diff --git a/lib/modules/manager/npm/post-update/npm.ts b/lib/modules/manager/npm/post-update/npm.ts index 728945fdb7da60..f0035106121ecf 100644 --- a/lib/modules/manager/npm/post-update/npm.ts +++ b/lib/modules/manager/npm/post-update/npm.ts @@ -234,7 +234,7 @@ export async function generateLockFile( if ( lockFileParsed.packages?.['']?.[depType]?.[lockUpdate.packageName!] ) { - lockFileParsed.packages[''][depType]![lockUpdate.packageName!] = + lockFileParsed.packages[''][depType][lockUpdate.packageName!] = lockUpdate.newValue!; } }); diff --git a/lib/modules/manager/nuget/artifacts.ts b/lib/modules/manager/nuget/artifacts.ts index 88f4a96e486983..c60e04262e6363 100644 --- a/lib/modules/manager/nuget/artifacts.ts +++ b/lib/modules/manager/nuget/artifacts.ts @@ -163,7 +163,7 @@ export async function updateArtifacts({ file: { type: 'addition', path: lockFileName, - contents: newLockFileContentMap[lockFileName]!, + contents: newLockFileContentMap[lockFileName], }, }); } From 37ed466a969ce47e37aca4c1f775465efc42265f Mon Sep 17 00:00:00 2001 From: secustor Date: Fri, 12 Jul 2024 09:27:06 +0200 Subject: [PATCH 2/2] revert automatic linter changes --- lib/config/migration.ts | 2 +- lib/modules/datasource/npm/npmrc.ts | 2 +- lib/modules/manager/gradle/parser/handlers.ts | 2 +- lib/modules/manager/npm/post-update/npm.ts | 2 +- lib/modules/manager/nuget/artifacts.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/config/migration.ts b/lib/config/migration.ts index 32317d4a41e81d..67f7d7a9035233 100644 --- a/lib/config/migration.ts +++ b/lib/config/migration.ts @@ -70,7 +70,7 @@ export function migrateConfig(config: RenovateConfig): MigratedConfig { } else if (is.array(val)) { if (is.array(migratedConfig?.[key])) { const newArray = []; - for (const item of migratedConfig[key]) { + for (const item of migratedConfig[key] as unknown[]) { if (is.object(item) && !is.array(item)) { const arrMigrate = migrateConfig(item as RenovateConfig); newArray.push(arrMigrate.migratedConfig); diff --git a/lib/modules/datasource/npm/npmrc.ts b/lib/modules/datasource/npm/npmrc.ts index 3e44ce476c371d..73b2e1eef52896 100644 --- a/lib/modules/datasource/npm/npmrc.ts +++ b/lib/modules/datasource/npm/npmrc.ts @@ -29,7 +29,7 @@ function envReplace(value: any, env = process.env): any { logger.warn('Failed to replace env in config: ' + match); throw new Error('env-replace'); } - return env[envVarName]; + return env[envVarName]!; }); } diff --git a/lib/modules/manager/gradle/parser/handlers.ts b/lib/modules/manager/gradle/parser/handlers.ts index 2d199f51eb3840..be12911f0eed9d 100644 --- a/lib/modules/manager/gradle/parser/handlers.ts +++ b/lib/modules/manager/gradle/parser/handlers.ts @@ -371,7 +371,7 @@ export function handleApplyFrom(ctx: Ctx): Ctx { const matchResult = parseGradle( // TODO #22198 - ctx.fileContents[scriptFilePath], + ctx.fileContents[scriptFilePath]!, ctx.globalVars, scriptFilePath, ctx.fileContents, diff --git a/lib/modules/manager/npm/post-update/npm.ts b/lib/modules/manager/npm/post-update/npm.ts index f0035106121ecf..728945fdb7da60 100644 --- a/lib/modules/manager/npm/post-update/npm.ts +++ b/lib/modules/manager/npm/post-update/npm.ts @@ -234,7 +234,7 @@ export async function generateLockFile( if ( lockFileParsed.packages?.['']?.[depType]?.[lockUpdate.packageName!] ) { - lockFileParsed.packages[''][depType][lockUpdate.packageName!] = + lockFileParsed.packages[''][depType]![lockUpdate.packageName!] = lockUpdate.newValue!; } }); diff --git a/lib/modules/manager/nuget/artifacts.ts b/lib/modules/manager/nuget/artifacts.ts index c60e04262e6363..88f4a96e486983 100644 --- a/lib/modules/manager/nuget/artifacts.ts +++ b/lib/modules/manager/nuget/artifacts.ts @@ -163,7 +163,7 @@ export async function updateArtifacts({ file: { type: 'addition', path: lockFileName, - contents: newLockFileContentMap[lockFileName], + contents: newLockFileContentMap[lockFileName]!, }, }); }