From ced823771a0a0be75d73a324534ef14246b996e5 Mon Sep 17 00:00:00 2001 From: secustor Date: Sun, 24 Oct 2021 11:14:41 +0200 Subject: [PATCH 1/5] feat(manager/regex): allow arbitrary regex groups for templates --- lib/manager/regex/__fixtures__/ansible.yml | 4 +- .../regex/__snapshots__/index.spec.ts.snap | 22 +++ lib/manager/regex/index.spec.ts | 23 +++ lib/manager/regex/index.ts | 155 +++--------------- lib/manager/regex/util.ts | 98 +++++++++++ 5 files changed, 168 insertions(+), 134 deletions(-) create mode 100644 lib/manager/regex/util.ts diff --git a/lib/manager/regex/__fixtures__/ansible.yml b/lib/manager/regex/__fixtures__/ansible.yml index b3647cd99fbeb1..f3cf9c112c6790 100644 --- a/lib/manager/regex/__fixtures__/ansible.yml +++ b/lib/manager/regex/__fixtures__/ansible.yml @@ -1,5 +1,7 @@ prometheus_image: "prom/prometheus" // depName gets initially set + +prometheus_registry: "docker.io" // depName gets initially set +prometheus_repository: "prom/prometheus" // depName gets initially set prometheus_version: "v2.21.0" // currentValue get set -someother_image: "" // will not be set as group value is null/empty string someother_version: "0.12.0" // overwrites currentValue as later values take precedence. diff --git a/lib/manager/regex/__snapshots__/index.spec.ts.snap b/lib/manager/regex/__snapshots__/index.spec.ts.snap index bcc3505484bc79..c2d1cdd76cde96 100644 --- a/lib/manager/regex/__snapshots__/index.spec.ts.snap +++ b/lib/manager/regex/__snapshots__/index.spec.ts.snap @@ -257,6 +257,28 @@ Object { } `; +exports[`manager/regex/index extracts with combination strategy and non standard capture groups 1`] = ` +Object { + "datasourceTemplate": "docker", + "depNameTemplate": "{{{ registry }}}/{{{ repository }}}", + "deps": Array [ + Object { + "currentValue": "v2.21.0", + "datasource": "docker", + "depName": "docker.io/prom/prometheus", + "replaceString": "prometheus_version: \\"v2.21.0\\" //", + }, + ], + "matchStrings": Array [ + "prometheus_registry:\\\\s*\\"(?.*)\\"\\\\s*\\\\/\\\\/", + "prometheus_repository:\\\\s*\\"(?.*)\\"\\\\s*\\\\/\\\\/", + "prometheus_tag:\\\\s*\\"(?.*)\\"\\\\s*\\\\/\\\\/", + "prometheus_version:\\\\s*\\"(?.*)\\"\\\\s*\\\\/\\\\/", + ], + "matchStringsStrategy": "combination", +} +`; + exports[`manager/regex/index extracts with combination strategy and registry url 1`] = ` Object { "datasourceTemplate": "helm", diff --git a/lib/manager/regex/index.spec.ts b/lib/manager/regex/index.spec.ts index 13258f85f2d53d..78065b80a81f0a 100644 --- a/lib/manager/regex/index.spec.ts +++ b/lib/manager/regex/index.spec.ts @@ -206,6 +206,29 @@ describe('manager/regex/index', () => { expect(res).toMatchSnapshot(); expect(res.deps).toHaveLength(1); }); + + it('extracts with combination strategy and non standard capture groups', async () => { + const config: CustomExtractConfig = { + matchStrings: [ + 'prometheus_registry:\\s*"(?.*)"\\s*\\/\\/', + 'prometheus_repository:\\s*"(?.*)"\\s*\\/\\/', + 'prometheus_tag:\\s*"(?.*)"\\s*\\/\\/', + 'prometheus_version:\\s*"(?.*)"\\s*\\/\\/', + ], + matchStringsStrategy: 'combination', + datasourceTemplate: 'docker', + depNameTemplate: '{{{ registry }}}/{{{ repository }}}', + }; + const res = await extractPackageFile( + ansibleYamlContent, + 'ansible.yml', + config + ); + expect(res.deps).toHaveLength(1); + expect(res.deps[0].depName).toEqual('docker.io/prom/prometheus'); + expect(res).toMatchSnapshot(); + }); + it('extracts with combination strategy and multiple matches', async () => { const config: CustomExtractConfig = { matchStrings: [ diff --git a/lib/manager/regex/index.ts b/lib/manager/regex/index.ts index 7a866a6f70e432..72f0d8cb8a7472 100644 --- a/lib/manager/regex/index.ts +++ b/lib/manager/regex/index.ts @@ -1,111 +1,21 @@ -import { URL } from 'url'; -import { logger } from '../../logger'; import { regEx } from '../../util/regex'; -import * as template from '../../util/template'; import type { CustomExtractConfig, PackageDependency, PackageFile, Result, } from '../types'; +import { + createDependency, + mergeExtractionTemplate, + regexMatchAll, + validMatchFields, +} from './util'; export const defaultConfig = { pinDigests: false, }; -const validMatchFields = [ - 'depName', - 'lookupName', - 'currentValue', - 'currentDigest', - 'datasource', - 'versioning', - 'extractVersion', - 'registryUrl', - 'depType', -]; - -const mergeFields = ['registryUrls', ...validMatchFields]; - -function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] { - const matches: RegExpMatchArray[] = []; - let matchResult; - do { - matchResult = regex.exec(content); - if (matchResult) { - matches.push(matchResult); - } - } while (matchResult); - return matches; -} - -function createDependency( - matchResult: RegExpMatchArray, - combinedGroups: Record, - config: CustomExtractConfig, - dep?: PackageDependency -): PackageDependency { - const dependency = dep || {}; - const { groups } = matchResult; - - function updateDependency(field: string, value: string): void { - switch (field) { - case 'registryUrl': - // check if URL is valid and pack inside an array - try { - const url = new URL(value).toString(); - dependency.registryUrls = [url]; - } catch (err) { - logger.warn({ value }, 'Invalid regex manager registryUrl'); - } - break; - default: - dependency[field] = value; - break; - } - } - - for (const field of validMatchFields) { - const fieldTemplate = `${field}Template`; - if (config[fieldTemplate]) { - try { - const compiled = template.compile( - config[fieldTemplate], - combinedGroups ?? groups, - false - ); - updateDependency(field, compiled); - } catch (err) { - logger.warn( - { template: config[fieldTemplate] }, - 'Error compiling template for custom manager' - ); - return null; - } - } else if (groups[field]) { - updateDependency(field, groups[field]); - } - } - dependency.replaceString = String(matchResult[0]); - return dependency; -} - -function mergeDependency(deps: PackageDependency[]): PackageDependency { - const result: PackageDependency = {}; - deps.forEach((dep) => { - mergeFields.forEach((field) => { - if (dep[field]) { - result[field] = dep[field]; - // save the line replaceString of the section which contains the current Value for a speed up lookup during the replace phase - if (field === 'currentValue') { - result.replaceString = dep.replaceString; - } - } - }); - }); - return result; -} - function handleAny( content: string, packageFile: string, @@ -114,29 +24,12 @@ function handleAny( return config.matchStrings .map((matchString) => regEx(matchString, 'g')) .flatMap((regex) => regexMatchAll(regex, content)) // match all regex to content, get all matches, reduce to single array - .map((matchResult) => createDependency(matchResult, null, config)); -} - -function mergeGroups( - mergedGroup: Record, - secondGroup: Record -): Record { - const resultGroup = {}; - - Object.keys(mergedGroup) - .filter((key) => validMatchFields.includes(key)) // prevent prototype pollution - .forEach( - // eslint-disable-next-line no-return-assign - (key) => (resultGroup[key] = mergedGroup[key]) + .map((matchResult) => + createDependency( + { groups: matchResult.groups, replaceString: matchResult[0] }, + config + ) ); - Object.keys(secondGroup) - .filter((key) => validMatchFields.includes(key)) // prevent prototype pollution - .forEach((key) => { - if (secondGroup[key] && secondGroup[key] !== '') { - resultGroup[key] = secondGroup[key]; - } - }); - return resultGroup; } function handleCombination( @@ -152,20 +45,13 @@ function handleCombination( return []; } - const combinedGroup = matches - .map((match) => match.groups) - .reduce((mergedGroup, currentGroup) => - mergeGroups(mergedGroup, currentGroup) - ); - - // TODO: this seems to be buggy behavior, needs to be checked #11387 - const dep = matches - .map((match) => createDependency(match, combinedGroup, config)) - .reduce( - (mergedDep, currentDep) => mergeDependency([mergedDep, currentDep]), - {} - ); // merge fields of dependencies - return [dep]; + const extraction = matches + .map((match) => ({ + groups: match.groups, + replaceString: match?.groups?.currentValue ? match[0] : undefined, + })) + .reduce((base, addition) => mergeExtractionTemplate(base, addition)); + return [createDependency(extraction, config)]; } function handleRecursive( @@ -184,7 +70,10 @@ function handleRecursive( return regexMatchAll(regexes[index], content).flatMap((match) => { // if we have a depName and a currentValue with have the minimal viable definition if (match?.groups?.depName && match?.groups?.currentValue) { - return createDependency(match, null, config); + return createDependency( + { groups: match.groups, replaceString: match[0] }, + config + ); } return handleRecursive(match[0], packageFile, config, index + 1); }); diff --git a/lib/manager/regex/util.ts b/lib/manager/regex/util.ts new file mode 100644 index 00000000000000..0568e2a69b79de --- /dev/null +++ b/lib/manager/regex/util.ts @@ -0,0 +1,98 @@ +import { URL } from 'url'; +import { logger } from '../../logger'; +import * as template from '../../util/template'; +import { CustomExtractConfig, PackageDependency } from '../types'; + +export const validMatchFields: string[] = [ + 'depName', + 'lookupName', + 'currentValue', + 'currentDigest', + 'datasource', + 'versioning', + 'extractVersion', + 'registryUrl', +]; + +interface ExtractionTemplate { + groups: Record; + replaceString: string; +} + +export function mergeGroups( + mergedGroup: Record, + secondGroup: Record +): Record { + return { ...mergedGroup, ...secondGroup }; +} + +export function mergeExtractionTemplate( + base: ExtractionTemplate, + addition: ExtractionTemplate +): ExtractionTemplate { + return { + groups: mergeGroups(base.groups, addition.groups), + replaceString: addition.replaceString ?? base.replaceString, + }; +} + +export function regexMatchAll( + regex: RegExp, + content: string +): RegExpMatchArray[] { + const matches: RegExpMatchArray[] = []; + let matchResult; + do { + matchResult = regex.exec(content); + if (matchResult) { + matches.push(matchResult); + } + } while (matchResult); + return matches; +} + +export function createDependency( + extractionTemplate: ExtractionTemplate, + config: CustomExtractConfig, + dep?: PackageDependency +): PackageDependency { + const dependency = dep || {}; + const { groups, replaceString } = extractionTemplate; + + function updateDependency(field: string, value: string): void { + switch (field) { + case 'registryUrl': + // check if URL is valid and pack inside an array + try { + const url = new URL(value).toString(); + dependency.registryUrls = [url]; + } catch (err) { + logger.warn({ value }, 'Invalid regex manager registryUrl'); + } + break; + default: + dependency[field] = value; + break; + } + } + + for (const field of validMatchFields) { + const fieldTemplate = `${field}Template`; + if (config[fieldTemplate]) { + try { + const compiled = template.compile(config[fieldTemplate], groups, false); + updateDependency(field, compiled); + } catch (err) { + logger.warn( + { template: config[fieldTemplate] }, + 'Error compiling template for custom manager' + ); + return null; + } + } else if (groups[field]) { + updateDependency(field, groups[field]); + } + } + dependency.replaceString = replaceString; + return dependency; +} From 1765280fefddffbe50658f7778a0d67ab0562d24 Mon Sep 17 00:00:00 2001 From: secustor Date: Mon, 25 Oct 2021 10:58:42 +0200 Subject: [PATCH 2/5] feat(manager/regex): allow arbitrary regex groups for templates --- lib/manager/regex/util.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/manager/regex/util.ts b/lib/manager/regex/util.ts index 0568e2a69b79de..7307a7954c015f 100644 --- a/lib/manager/regex/util.ts +++ b/lib/manager/regex/util.ts @@ -12,6 +12,7 @@ export const validMatchFields: string[] = [ 'versioning', 'extractVersion', 'registryUrl', + 'depType', ]; interface ExtractionTemplate { From 1abee43ccae9f4269530a17e0a4b6e00e11ee818 Mon Sep 17 00:00:00 2001 From: secustor Date: Sun, 31 Oct 2021 10:50:04 +0100 Subject: [PATCH 3/5] chore fix linting issue --- lib/manager/regex/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/manager/regex/index.ts b/lib/manager/regex/index.ts index 482e71be3a72d5..89aba3d1cef782 100644 --- a/lib/manager/regex/index.ts +++ b/lib/manager/regex/index.ts @@ -8,8 +8,8 @@ import type { import { createDependency, mergeExtractionTemplate, - regexMatchAll, mergeGroups, + regexMatchAll, validMatchFields, } from './util'; From bcc7bb23ab3602c4bf8f3f571c339983a34f5376 Mon Sep 17 00:00:00 2001 From: secustor Date: Sun, 31 Oct 2021 19:02:54 +0100 Subject: [PATCH 4/5] revert migration to util.ts --- lib/manager/regex/index.ts | 102 ++++++++++++++++++++++++++++++++++--- lib/manager/regex/util.ts | 99 ----------------------------------- 2 files changed, 95 insertions(+), 106 deletions(-) delete mode 100644 lib/manager/regex/util.ts diff --git a/lib/manager/regex/index.ts b/lib/manager/regex/index.ts index 89aba3d1cef782..fef4994eddad58 100644 --- a/lib/manager/regex/index.ts +++ b/lib/manager/regex/index.ts @@ -1,22 +1,93 @@ +import { URL } from 'url'; +import { logger } from '../../logger'; import { regEx } from '../../util/regex'; +import * as template from '../../util/template'; import type { CustomExtractConfig, PackageDependency, PackageFile, Result, } from '../types'; -import { - createDependency, - mergeExtractionTemplate, - mergeGroups, - regexMatchAll, - validMatchFields, -} from './util'; export const defaultConfig = { pinDigests: false, }; +const validMatchFields = [ + 'depName', + 'lookupName', + 'currentValue', + 'currentDigest', + 'datasource', + 'versioning', + 'extractVersion', + 'registryUrl', + 'depType', +]; + +interface ExtractionTemplate { + groups: Record; + replaceString: string; +} + +function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] { + const matches: RegExpMatchArray[] = []; + let matchResult; + do { + matchResult = regex.exec(content); + if (matchResult) { + matches.push(matchResult); + } + } while (matchResult); + return matches; +} + +function createDependency( + extractionTemplate: ExtractionTemplate, + config: CustomExtractConfig, + dep?: PackageDependency +): PackageDependency { + const dependency = dep || {}; + const { groups, replaceString } = extractionTemplate; + + function updateDependency(field: string, value: string): void { + switch (field) { + case 'registryUrl': + // check if URL is valid and pack inside an array + try { + const url = new URL(value).toString(); + dependency.registryUrls = [url]; + } catch (err) { + logger.warn({ value }, 'Invalid regex manager registryUrl'); + } + break; + default: + dependency[field] = value; + break; + } + } + + for (const field of validMatchFields) { + const fieldTemplate = `${field}Template`; + if (config[fieldTemplate]) { + try { + const compiled = template.compile(config[fieldTemplate], groups, false); + updateDependency(field, compiled); + } catch (err) { + logger.warn( + { template: config[fieldTemplate] }, + 'Error compiling template for custom manager' + ); + return null; + } + } else if (groups[field]) { + updateDependency(field, groups[field]); + } + } + dependency.replaceString = replaceString; + return dependency; +} + function handleAny( content: string, packageFile: string, @@ -33,6 +104,23 @@ function handleAny( ); } +function mergeGroups( + mergedGroup: Record, + secondGroup: Record +): Record { + return { ...mergedGroup, ...secondGroup }; +} + +export function mergeExtractionTemplate( + base: ExtractionTemplate, + addition: ExtractionTemplate +): ExtractionTemplate { + return { + groups: mergeGroups(base.groups, addition.groups), + replaceString: addition.replaceString ?? base.replaceString, + }; +} + function handleCombination( content: string, packageFile: string, diff --git a/lib/manager/regex/util.ts b/lib/manager/regex/util.ts deleted file mode 100644 index 7307a7954c015f..00000000000000 --- a/lib/manager/regex/util.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { URL } from 'url'; -import { logger } from '../../logger'; -import * as template from '../../util/template'; -import { CustomExtractConfig, PackageDependency } from '../types'; - -export const validMatchFields: string[] = [ - 'depName', - 'lookupName', - 'currentValue', - 'currentDigest', - 'datasource', - 'versioning', - 'extractVersion', - 'registryUrl', - 'depType', -]; - -interface ExtractionTemplate { - groups: Record; - replaceString: string; -} - -export function mergeGroups( - mergedGroup: Record, - secondGroup: Record -): Record { - return { ...mergedGroup, ...secondGroup }; -} - -export function mergeExtractionTemplate( - base: ExtractionTemplate, - addition: ExtractionTemplate -): ExtractionTemplate { - return { - groups: mergeGroups(base.groups, addition.groups), - replaceString: addition.replaceString ?? base.replaceString, - }; -} - -export function regexMatchAll( - regex: RegExp, - content: string -): RegExpMatchArray[] { - const matches: RegExpMatchArray[] = []; - let matchResult; - do { - matchResult = regex.exec(content); - if (matchResult) { - matches.push(matchResult); - } - } while (matchResult); - return matches; -} - -export function createDependency( - extractionTemplate: ExtractionTemplate, - config: CustomExtractConfig, - dep?: PackageDependency -): PackageDependency { - const dependency = dep || {}; - const { groups, replaceString } = extractionTemplate; - - function updateDependency(field: string, value: string): void { - switch (field) { - case 'registryUrl': - // check if URL is valid and pack inside an array - try { - const url = new URL(value).toString(); - dependency.registryUrls = [url]; - } catch (err) { - logger.warn({ value }, 'Invalid regex manager registryUrl'); - } - break; - default: - dependency[field] = value; - break; - } - } - - for (const field of validMatchFields) { - const fieldTemplate = `${field}Template`; - if (config[fieldTemplate]) { - try { - const compiled = template.compile(config[fieldTemplate], groups, false); - updateDependency(field, compiled); - } catch (err) { - logger.warn( - { template: config[fieldTemplate] }, - 'Error compiling template for custom manager' - ); - return null; - } - } else if (groups[field]) { - updateDependency(field, groups[field]); - } - } - dependency.replaceString = replaceString; - return dependency; -} From 9f65deb9caaa98a52e9500779eb2d48dffb279d5 Mon Sep 17 00:00:00 2001 From: secustor Date: Mon, 1 Nov 2021 20:50:21 +0100 Subject: [PATCH 5/5] move interface to types.ts --- lib/manager/regex/index.ts | 6 +----- lib/manager/regex/types.ts | 4 ++++ 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 lib/manager/regex/types.ts diff --git a/lib/manager/regex/index.ts b/lib/manager/regex/index.ts index fef4994eddad58..0facc64a35de42 100644 --- a/lib/manager/regex/index.ts +++ b/lib/manager/regex/index.ts @@ -8,6 +8,7 @@ import type { PackageFile, Result, } from '../types'; +import type { ExtractionTemplate } from './types'; export const defaultConfig = { pinDigests: false, @@ -25,11 +26,6 @@ const validMatchFields = [ 'depType', ]; -interface ExtractionTemplate { - groups: Record; - replaceString: string; -} - function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] { const matches: RegExpMatchArray[] = []; let matchResult; diff --git a/lib/manager/regex/types.ts b/lib/manager/regex/types.ts new file mode 100644 index 00000000000000..f75929b91288c5 --- /dev/null +++ b/lib/manager/regex/types.ts @@ -0,0 +1,4 @@ +export interface ExtractionTemplate { + groups: Record; + replaceString: string; +}