From d7ead66057384c2fb8d3d871aa864596c4c11177 Mon Sep 17 00:00:00 2001 From: secustor Date: Sun, 24 Oct 2021 11:14:41 +0200 Subject: [PATCH] feat(manager/regex): allow arbitrary regex groups for templates --- lib/manager/regex/__fixtures__/ansible.yml | 4 +- .../regex/__snapshots__/index.spec.ts.snap | 22 +++ lib/manager/regex/index.spec.ts | 23 +++ lib/manager/regex/index.ts | 154 +++--------------- lib/manager/regex/util.ts | 98 +++++++++++ 5 files changed, 168 insertions(+), 133 deletions(-) create mode 100644 lib/manager/regex/util.ts diff --git a/lib/manager/regex/__fixtures__/ansible.yml b/lib/manager/regex/__fixtures__/ansible.yml index b3647cd99fbeb1b..f3cf9c112c67907 100644 --- a/lib/manager/regex/__fixtures__/ansible.yml +++ b/lib/manager/regex/__fixtures__/ansible.yml @@ -1,5 +1,7 @@ prometheus_image: "prom/prometheus" // depName gets initially set + +prometheus_registry: "docker.io" // depName gets initially set +prometheus_repository: "prom/prometheus" // depName gets initially set prometheus_version: "v2.21.0" // currentValue get set -someother_image: "" // will not be set as group value is null/empty string someother_version: "0.12.0" // overwrites currentValue as later values take precedence. diff --git a/lib/manager/regex/__snapshots__/index.spec.ts.snap b/lib/manager/regex/__snapshots__/index.spec.ts.snap index 6071a39e3525252..1090b598a8529e9 100644 --- a/lib/manager/regex/__snapshots__/index.spec.ts.snap +++ b/lib/manager/regex/__snapshots__/index.spec.ts.snap @@ -248,6 +248,28 @@ Object { } `; +exports[`manager/regex/index extracts with combination strategy and non standard capture groups 1`] = ` +Object { + "datasourceTemplate": "docker", + "depNameTemplate": "{{{ registry }}}/{{{ repository }}}", + "deps": Array [ + Object { + "currentValue": "v2.21.0", + "datasource": "docker", + "depName": "docker.io/prom/prometheus", + "replaceString": "prometheus_version: \\"v2.21.0\\" //", + }, + ], + "matchStrings": Array [ + "prometheus_registry:\\\\s*\\"(?.*)\\"\\\\s*\\\\/\\\\/", + "prometheus_repository:\\\\s*\\"(?.*)\\"\\\\s*\\\\/\\\\/", + "prometheus_tag:\\\\s*\\"(?.*)\\"\\\\s*\\\\/\\\\/", + "prometheus_version:\\\\s*\\"(?.*)\\"\\\\s*\\\\/\\\\/", + ], + "matchStringsStrategy": "combination", +} +`; + exports[`manager/regex/index extracts with combination strategy and registry url 1`] = ` Object { "datasourceTemplate": "helm", diff --git a/lib/manager/regex/index.spec.ts b/lib/manager/regex/index.spec.ts index 7f3d9bdfc68026b..ea6c55eab879f05 100644 --- a/lib/manager/regex/index.spec.ts +++ b/lib/manager/regex/index.spec.ts @@ -204,6 +204,29 @@ describe('manager/regex/index', () => { expect(res).toMatchSnapshot(); expect(res.deps).toHaveLength(1); }); + + it('extracts with combination strategy and non standard capture groups', async () => { + const config: CustomExtractConfig = { + matchStrings: [ + 'prometheus_registry:\\s*"(?.*)"\\s*\\/\\/', + 'prometheus_repository:\\s*"(?.*)"\\s*\\/\\/', + 'prometheus_tag:\\s*"(?.*)"\\s*\\/\\/', + 'prometheus_version:\\s*"(?.*)"\\s*\\/\\/', + ], + matchStringsStrategy: 'combination', + datasourceTemplate: 'docker', + depNameTemplate: '{{{ registry }}}/{{{ repository }}}', + }; + const res = await extractPackageFile( + ansibleYamlContent, + 'ansible.yml', + config + ); + expect(res.deps).toHaveLength(1); + expect(res.deps[0].depName).toEqual('docker.io/prom/prometheus'); + expect(res).toMatchSnapshot(); + }); + it('extracts with combination strategy and multiple matches', async () => { const config: CustomExtractConfig = { matchStrings: [ diff --git a/lib/manager/regex/index.ts b/lib/manager/regex/index.ts index 70f9c181ef11f50..72f0d8cb8a74724 100644 --- a/lib/manager/regex/index.ts +++ b/lib/manager/regex/index.ts @@ -1,110 +1,21 @@ -import { URL } from 'url'; -import { logger } from '../../logger'; import { regEx } from '../../util/regex'; -import * as template from '../../util/template'; import type { CustomExtractConfig, PackageDependency, PackageFile, Result, } from '../types'; +import { + createDependency, + mergeExtractionTemplate, + regexMatchAll, + validMatchFields, +} from './util'; export const defaultConfig = { pinDigests: false, }; -const validMatchFields = [ - 'depName', - 'lookupName', - 'currentValue', - 'currentDigest', - 'datasource', - 'versioning', - 'extractVersion', - 'registryUrl', -]; - -const mergeFields = ['registryUrls', ...validMatchFields]; - -function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] { - const matches: RegExpMatchArray[] = []; - let matchResult; - do { - matchResult = regex.exec(content); - if (matchResult) { - matches.push(matchResult); - } - } while (matchResult); - return matches; -} - -function createDependency( - matchResult: RegExpMatchArray, - combinedGroups: Record, - config: CustomExtractConfig, - dep?: PackageDependency -): PackageDependency { - const dependency = dep || {}; - const { groups } = matchResult; - - function updateDependency(field: string, value: string): void { - switch (field) { - case 'registryUrl': - // check if URL is valid and pack inside an array - try { - const url = new URL(value).toString(); - dependency.registryUrls = [url]; - } catch (err) { - logger.warn({ value }, 'Invalid regex manager registryUrl'); - } - break; - default: - dependency[field] = value; - break; - } - } - - for (const field of validMatchFields) { - const fieldTemplate = `${field}Template`; - if (config[fieldTemplate]) { - try { - const compiled = template.compile( - config[fieldTemplate], - combinedGroups ?? groups, - false - ); - updateDependency(field, compiled); - } catch (err) { - logger.warn( - { template: config[fieldTemplate] }, - 'Error compiling template for custom manager' - ); - return null; - } - } else if (groups[field]) { - updateDependency(field, groups[field]); - } - } - dependency.replaceString = String(matchResult[0]); - return dependency; -} - -function mergeDependency(deps: PackageDependency[]): PackageDependency { - const result: PackageDependency = {}; - deps.forEach((dep) => { - mergeFields.forEach((field) => { - if (dep[field]) { - result[field] = dep[field]; - // save the line replaceString of the section which contains the current Value for a speed up lookup during the replace phase - if (field === 'currentValue') { - result.replaceString = dep.replaceString; - } - } - }); - }); - return result; -} - function handleAny( content: string, packageFile: string, @@ -113,29 +24,12 @@ function handleAny( return config.matchStrings .map((matchString) => regEx(matchString, 'g')) .flatMap((regex) => regexMatchAll(regex, content)) // match all regex to content, get all matches, reduce to single array - .map((matchResult) => createDependency(matchResult, null, config)); -} - -function mergeGroups( - mergedGroup: Record, - secondGroup: Record -): Record { - const resultGroup = {}; - - Object.keys(mergedGroup) - .filter((key) => validMatchFields.includes(key)) // prevent prototype pollution - .forEach( - // eslint-disable-next-line no-return-assign - (key) => (resultGroup[key] = mergedGroup[key]) + .map((matchResult) => + createDependency( + { groups: matchResult.groups, replaceString: matchResult[0] }, + config + ) ); - Object.keys(secondGroup) - .filter((key) => validMatchFields.includes(key)) // prevent prototype pollution - .forEach((key) => { - if (secondGroup[key] && secondGroup[key] !== '') { - resultGroup[key] = secondGroup[key]; - } - }); - return resultGroup; } function handleCombination( @@ -151,20 +45,13 @@ function handleCombination( return []; } - const combinedGroup = matches - .map((match) => match.groups) - .reduce((mergedGroup, currentGroup) => - mergeGroups(mergedGroup, currentGroup) - ); - - // TODO: this seems to be buggy behavior, needs to be checked #11387 - const dep = matches - .map((match) => createDependency(match, combinedGroup, config)) - .reduce( - (mergedDep, currentDep) => mergeDependency([mergedDep, currentDep]), - {} - ); // merge fields of dependencies - return [dep]; + const extraction = matches + .map((match) => ({ + groups: match.groups, + replaceString: match?.groups?.currentValue ? match[0] : undefined, + })) + .reduce((base, addition) => mergeExtractionTemplate(base, addition)); + return [createDependency(extraction, config)]; } function handleRecursive( @@ -183,7 +70,10 @@ function handleRecursive( return regexMatchAll(regexes[index], content).flatMap((match) => { // if we have a depName and a currentValue with have the minimal viable definition if (match?.groups?.depName && match?.groups?.currentValue) { - return createDependency(match, null, config); + return createDependency( + { groups: match.groups, replaceString: match[0] }, + config + ); } return handleRecursive(match[0], packageFile, config, index + 1); }); diff --git a/lib/manager/regex/util.ts b/lib/manager/regex/util.ts new file mode 100644 index 000000000000000..0568e2a69b79de7 --- /dev/null +++ b/lib/manager/regex/util.ts @@ -0,0 +1,98 @@ +import { URL } from 'url'; +import { logger } from '../../logger'; +import * as template from '../../util/template'; +import { CustomExtractConfig, PackageDependency } from '../types'; + +export const validMatchFields: string[] = [ + 'depName', + 'lookupName', + 'currentValue', + 'currentDigest', + 'datasource', + 'versioning', + 'extractVersion', + 'registryUrl', +]; + +interface ExtractionTemplate { + groups: Record; + replaceString: string; +} + +export function mergeGroups( + mergedGroup: Record, + secondGroup: Record +): Record { + return { ...mergedGroup, ...secondGroup }; +} + +export function mergeExtractionTemplate( + base: ExtractionTemplate, + addition: ExtractionTemplate +): ExtractionTemplate { + return { + groups: mergeGroups(base.groups, addition.groups), + replaceString: addition.replaceString ?? base.replaceString, + }; +} + +export function regexMatchAll( + regex: RegExp, + content: string +): RegExpMatchArray[] { + const matches: RegExpMatchArray[] = []; + let matchResult; + do { + matchResult = regex.exec(content); + if (matchResult) { + matches.push(matchResult); + } + } while (matchResult); + return matches; +} + +export function createDependency( + extractionTemplate: ExtractionTemplate, + config: CustomExtractConfig, + dep?: PackageDependency +): PackageDependency { + const dependency = dep || {}; + const { groups, replaceString } = extractionTemplate; + + function updateDependency(field: string, value: string): void { + switch (field) { + case 'registryUrl': + // check if URL is valid and pack inside an array + try { + const url = new URL(value).toString(); + dependency.registryUrls = [url]; + } catch (err) { + logger.warn({ value }, 'Invalid regex manager registryUrl'); + } + break; + default: + dependency[field] = value; + break; + } + } + + for (const field of validMatchFields) { + const fieldTemplate = `${field}Template`; + if (config[fieldTemplate]) { + try { + const compiled = template.compile(config[fieldTemplate], groups, false); + updateDependency(field, compiled); + } catch (err) { + logger.warn( + { template: config[fieldTemplate] }, + 'Error compiling template for custom manager' + ); + return null; + } + } else if (groups[field]) { + updateDependency(field, groups[field]); + } + } + dependency.replaceString = replaceString; + return dependency; +}