feat(manager/regex)!: allow arbitrary regex groups for templates

lib/manager/regex/__fixtures__/ansible.yml

@@ -1,5 +1,7 @@
 prometheus_image: "prom/prometheus"  // depName gets initially set
+
+prometheus_registry: "docker.io"  // depName gets initially set
+prometheus_repository: "prom/prometheus"  // depName gets initially set
 prometheus_version: "v2.21.0" // currentValue get set
 
-someother_image: "" // will not be set as group value is null/empty string
 someother_version: "0.12.0" // overwrites currentValue as later values take precedence.

lib/manager/regex/__snapshots__/index.spec.ts.snap

@@ -257,6 +257,28 @@ Object {
 }
 `;
 
+exports[`manager/regex/index extracts with combination strategy and non standard capture groups 1`] = `
+Object {
+  "datasourceTemplate": "docker",
+  "depNameTemplate": "{{{ registry }}}/{{{ repository }}}",
+  "deps": Array [
+    Object {
+      "currentValue": "v2.21.0",
+      "datasource": "docker",
+      "depName": "docker.io/prom/prometheus",
+      "replaceString": "prometheus_version: \\"v2.21.0\\" //",
+    },
+  ],
+  "matchStrings": Array [
+    "prometheus_registry:\\\\s*\\"(?<registry>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_repository:\\\\s*\\"(?<repository>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_tag:\\\\s*\\"(?<tag>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_version:\\\\s*\\"(?<currentValue>.*)\\"\\\\s*\\\\/\\\\/",
+  ],
+  "matchStringsStrategy": "combination",
+}
+`;
+
 exports[`manager/regex/index extracts with combination strategy and registry url 1`] = `
 Object {
   "datasourceTemplate": "helm",

lib/manager/regex/index.spec.ts

@@ -206,6 +206,29 @@ describe('manager/regex/index', () => {
     expect(res).toMatchSnapshot();
     expect(res.deps).toHaveLength(1);
   });
+
+  it('extracts with combination strategy and non standard capture groups', async () => {
+    const config: CustomExtractConfig = {
+      matchStrings: [
+        'prometheus_registry:\\s*"(?<registry>.*)"\\s*\\/\\/',
+        'prometheus_repository:\\s*"(?<repository>.*)"\\s*\\/\\/',
+        'prometheus_tag:\\s*"(?<tag>.*)"\\s*\\/\\/',
+        'prometheus_version:\\s*"(?<currentValue>.*)"\\s*\\/\\/',
+      ],
+      matchStringsStrategy: 'combination',
+      datasourceTemplate: 'docker',
+      depNameTemplate: '{{{ registry }}}/{{{ repository }}}',
+    };
+    const res = await extractPackageFile(
+      ansibleYamlContent,
+      'ansible.yml',
+      config
+    );
+    expect(res.deps).toHaveLength(1);
+    expect(res.deps[0].depName).toEqual('docker.io/prom/prometheus');
+    expect(res).toMatchSnapshot();
+  });
+
   it('extracts with combination strategy and multiple matches', async () => {
     const config: CustomExtractConfig = {
       matchStrings: [

lib/manager/regex/index.ts

@@ -1,111 +1,21 @@
-import { URL } from 'url';
-import { logger } from '../../logger';
 import { regEx } from '../../util/regex';
-import * as template from '../../util/template';
 import type {
   CustomExtractConfig,
   PackageDependency,
   PackageFile,
   Result,
 } from '../types';
+import {
+  createDependency,
+  mergeExtractionTemplate,
+  regexMatchAll,
+  validMatchFields,
+} from './util';
 
 export const defaultConfig = {
   pinDigests: false,
 };
 
-const validMatchFields = [
-  'depName',
-  'lookupName',
-  'currentValue',
-  'currentDigest',
-  'datasource',
-  'versioning',
-  'extractVersion',
-  'registryUrl',
-  'depType',
-];
-
-const mergeFields = ['registryUrls', ...validMatchFields];
-
-function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] {
-  const matches: RegExpMatchArray[] = [];
-  let matchResult;
-  do {
-    matchResult = regex.exec(content);
-    if (matchResult) {
-      matches.push(matchResult);
-    }
-  } while (matchResult);
-  return matches;
-}
-
-function createDependency(
-  matchResult: RegExpMatchArray,
-  combinedGroups: Record<string, string>,
-  config: CustomExtractConfig,
-  dep?: PackageDependency
-): PackageDependency {
-  const dependency = dep || {};
-  const { groups } = matchResult;
-
-  function updateDependency(field: string, value: string): void {
-    switch (field) {
-      case 'registryUrl':
-        // check if URL is valid and pack inside an array
-        try {
-          const url = new URL(value).toString();
-          dependency.registryUrls = [url];
-        } catch (err) {
-          logger.warn({ value }, 'Invalid regex manager registryUrl');
-        }
-        break;
-      default:
-        dependency[field] = value;
-        break;
-    }
-  }
-
-  for (const field of validMatchFields) {
-    const fieldTemplate = `${field}Template`;
-    if (config[fieldTemplate]) {
-      try {
-        const compiled = template.compile(
-          config[fieldTemplate],
-          combinedGroups ?? groups,
-          false
-        );
-        updateDependency(field, compiled);
-      } catch (err) {
-        logger.warn(
-          { template: config[fieldTemplate] },
-          'Error compiling template for custom manager'
-        );
-        return null;
-      }
-    } else if (groups[field]) {
-      updateDependency(field, groups[field]);
-    }
-  }
-  dependency.replaceString = String(matchResult[0]);
-  return dependency;
-}
-
-function mergeDependency(deps: PackageDependency[]): PackageDependency {
-  const result: PackageDependency = {};
-  deps.forEach((dep) => {
-    mergeFields.forEach((field) => {
-      if (dep[field]) {
-        result[field] = dep[field];
-        // save the line replaceString of the section which contains the current Value for a speed up lookup during the replace phase
-        if (field === 'currentValue') {
-          result.replaceString = dep.replaceString;
-        }
-      }
-    });
-  });
-  return result;
-}
-
 function handleAny(
   content: string,
   packageFile: string,
@@ -114,29 +24,12 @@ function handleAny(
   return config.matchStrings
     .map((matchString) => regEx(matchString, 'g'))
     .flatMap((regex) => regexMatchAll(regex, content)) // match all regex to content, get all matches, reduce to single array
-    .map((matchResult) => createDependency(matchResult, null, config));
-}
-
-function mergeGroups(
-  mergedGroup: Record<string, string>,
-  secondGroup: Record<string, string>
-): Record<string, string> {
-  const resultGroup = {};
-
-  Object.keys(mergedGroup)
-    .filter((key) => validMatchFields.includes(key)) // prevent prototype pollution
-    .forEach(
-      // eslint-disable-next-line no-return-assign
-      (key) => (resultGroup[key] = mergedGroup[key])
+    .map((matchResult) =>
+      createDependency(
+        { groups: matchResult.groups, replaceString: matchResult[0] },
+        config
+      )
     );
-  Object.keys(secondGroup)
-    .filter((key) => validMatchFields.includes(key)) // prevent prototype pollution
-    .forEach((key) => {
-      if (secondGroup[key] && secondGroup[key] !== '') {
-        resultGroup[key] = secondGroup[key];
-      }
-    });
-  return resultGroup;
 }
 
 function handleCombination(
@@ -152,20 +45,13 @@ function handleCombination(
     return [];
   }
 
-  const combinedGroup = matches
-    .map((match) => match.groups)
-    .reduce((mergedGroup, currentGroup) =>
-      mergeGroups(mergedGroup, currentGroup)
-    );
-
-  // TODO: this seems to be buggy behavior, needs to be checked #11387
-  const dep = matches
-    .map((match) => createDependency(match, combinedGroup, config))
-    .reduce(
-      (mergedDep, currentDep) => mergeDependency([mergedDep, currentDep]),
-      {}
-    ); // merge fields of dependencies
-  return [dep];
+  const extraction = matches
+    .map((match) => ({
+      groups: match.groups,
+      replaceString: match?.groups?.currentValue ? match[0] : undefined,
+    }))
+    .reduce((base, addition) => mergeExtractionTemplate(base, addition));
+  return [createDependency(extraction, config)];
 }
 
 function handleRecursive(
@@ -184,7 +70,10 @@ function handleRecursive(
   return regexMatchAll(regexes[index], content).flatMap((match) => {
     // if we have a depName and a currentValue with have the minimal viable definition
     if (match?.groups?.depName && match?.groups?.currentValue) {
-      return createDependency(match, null, config);
+      return createDependency(
+        { groups: match.groups, replaceString: match[0] },
+        config
+      );
     }
     return handleRecursive(match[0], packageFile, config, index + 1);
   });

lib/manager/regex/util.ts

@@ -0,0 +1,99 @@
+import { URL } from 'url';
+import { logger } from '../../logger';
+import * as template from '../../util/template';
+import { CustomExtractConfig, PackageDependency } from '../types';
+
+export const validMatchFields: string[] = [
+  'depName',
+  'lookupName',
+  'currentValue',
+  'currentDigest',
+  'datasource',
+  'versioning',
+  'extractVersion',
+  'registryUrl',
+  'depType',
+];
+
+interface ExtractionTemplate {
+  groups: Record<string, string>;
+  replaceString: string;
+}
+
+export function mergeGroups(
+  mergedGroup: Record<string, string>,
+  secondGroup: Record<string, string>
+): Record<string, string> {
+  return { ...mergedGroup, ...secondGroup };
+}
+
+export function mergeExtractionTemplate(
+  base: ExtractionTemplate,
+  addition: ExtractionTemplate
+): ExtractionTemplate {
+  return {
+    groups: mergeGroups(base.groups, addition.groups),
+    replaceString: addition.replaceString ?? base.replaceString,
+  };
+}
+
+export function regexMatchAll(
+  regex: RegExp,
+  content: string
+): RegExpMatchArray[] {
+  const matches: RegExpMatchArray[] = [];
+  let matchResult;
+  do {
+    matchResult = regex.exec(content);
+    if (matchResult) {
+      matches.push(matchResult);
+    }
+  } while (matchResult);
+  return matches;
+}
+
+export function createDependency(
+  extractionTemplate: ExtractionTemplate,
+  config: CustomExtractConfig,
+  dep?: PackageDependency
+): PackageDependency {
+  const dependency = dep || {};
+  const { groups, replaceString } = extractionTemplate;
+
+  function updateDependency(field: string, value: string): void {
+    switch (field) {
+      case 'registryUrl':
+        // check if URL is valid and pack inside an array
+        try {
+          const url = new URL(value).toString();
+          dependency.registryUrls = [url];
+        } catch (err) {
+          logger.warn({ value }, 'Invalid regex manager registryUrl');
+        }
+        break;
+      default:
+        dependency[field] = value;
+        break;
+    }
+  }
+
+  for (const field of validMatchFields) {
+    const fieldTemplate = `${field}Template`;
+    if (config[fieldTemplate]) {
+      try {
+        const compiled = template.compile(config[fieldTemplate], groups, false);
+        updateDependency(field, compiled);
+      } catch (err) {
+        logger.warn(
+          { template: config[fieldTemplate] },
+          'Error compiling template for custom manager'
+        );
+        return null;
+      }
+    } else if (groups[field]) {
+      updateDependency(field, groups[field]);
+    }
+  }
+  dependency.replaceString = replaceString;
+  return dependency;
+}