From 9806024f0540f1de499a4fbca1258dedacde4d5e Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Wed, 11 Dec 2024 15:06:43 +0100 Subject: [PATCH 1/4] Add `scripts/relocate` CLI (beta) --- package.json | 1 + packages/kbn-relocate/README.md | 64 +++++++ packages/kbn-relocate/constants.ts | 99 +++++++++++ packages/kbn-relocate/index.ts | 77 +++++++++ packages/kbn-relocate/jest.config.js | 14 ++ packages/kbn-relocate/kibana.jsonc | 6 + packages/kbn-relocate/package.json | 6 + packages/kbn-relocate/relocate.ts | 213 ++++++++++++++++++++++++ packages/kbn-relocate/transforms.ts | 49 ++++++ packages/kbn-relocate/tsconfig.json | 23 +++ packages/kbn-relocate/types.ts | 24 +++ packages/kbn-relocate/utils.exec.ts | 35 ++++ packages/kbn-relocate/utils.git.ts | 91 ++++++++++ packages/kbn-relocate/utils.logging.ts | 107 ++++++++++++ packages/kbn-relocate/utils.relocate.ts | 210 +++++++++++++++++++++++ scripts/relocate.js | 11 ++ tsconfig.base.json | 2 + yarn.lock | 4 + 18 files changed, 1036 insertions(+) create mode 100644 packages/kbn-relocate/README.md create mode 100644 packages/kbn-relocate/constants.ts create mode 100644 packages/kbn-relocate/index.ts create mode 100644 packages/kbn-relocate/jest.config.js create mode 100644 packages/kbn-relocate/kibana.jsonc create mode 100644 packages/kbn-relocate/package.json create mode 100644 packages/kbn-relocate/relocate.ts create mode 100644 packages/kbn-relocate/transforms.ts create mode 100644 packages/kbn-relocate/tsconfig.json create mode 100644 packages/kbn-relocate/types.ts create mode 100644 packages/kbn-relocate/utils.exec.ts create mode 100644 packages/kbn-relocate/utils.git.ts create mode 100644 packages/kbn-relocate/utils.logging.ts create mode 100644 packages/kbn-relocate/utils.relocate.ts create mode 100644 scripts/relocate.js diff --git a/package.json b/package.json index 4c1722a904255..9081ece7803ee 100644 --- a/package.json +++ b/package.json @@ -1490,6 +1490,7 @@ "@kbn/plugin-generator": "link:packages/kbn-plugin-generator", "@kbn/plugin-helpers": "link:packages/kbn-plugin-helpers", "@kbn/product-doc-artifact-builder": "link:x-pack/packages/ai-infra/product-doc-artifact-builder", + "@kbn/relocate": "link:packages/kbn-relocate", "@kbn/repo-file-maps": "link:packages/kbn-repo-file-maps", "@kbn/repo-linter": "link:packages/kbn-repo-linter", "@kbn/repo-path": "link:packages/kbn-repo-path", diff --git a/packages/kbn-relocate/README.md b/packages/kbn-relocate/README.md new file mode 100644 index 0000000000000..2edba006f7944 --- /dev/null +++ b/packages/kbn-relocate/README.md @@ -0,0 +1,64 @@ +# @kbn/relocate + +This package contains a CLI tool to help move modules (plugins and packages) into their intended folders, according to the _Sustainable Kibana Architecture. + +## Prerequisites + +You must have `gh` CLI tool installed. You can install it by running: + +```sh +brew install gh +``` + +You must also configure your "default" kibana repo in `gh`, so that it can find PRs.: + +```sh +gh repo set-default elastic/kibana +``` + +You must have `elastic/kibana` remote configured under the name `upstream`. + +## Usage + +First of all, you need to decide whether you want to contribute to an existing PR or to create a new one. Use the `--pr` flag to specify the PR you are trying to update: + +```sh +node scripts/relocate --pr +``` + +Note that when specifying an existing PR, the logic will undo + rewrite history for that PR, by force-pushing changes. + +To relocate modules for a given team, identify the "team handle" (e.g. @elastic/kibana-core), and run the following command from the root of the Kibana repo: + +```sh +node scripts/relocate --pr --team +``` + +You can relocate modules by path, e.g. all modules that are under `x-pack/plugins/observability_solution/`: + +```sh +node scripts/relocate --pr --path "x-pack/plugins/observability_solution/" +``` + +You can specify indivual packages by ID: + +```sh +node scripts/relocate --pr --include "@kbn/data-forge" --include "@kbn/deeplinks-observability" +``` + +You can also specify combinations of the above filters, to include modules that match ANY of the criteria. +Excluding modules explictly is also supported: + +```sh +node scripts/relocate --pr --team "@elastic/obs-ux-management-team" --exclude "@kbn/data-forge" +``` + +## Details + +The script generates log / description files of the form `relocate_YYYYMMDDhhmmss_.out`. You can inspect them if you encounter any errors. + +In particular, the file `relocate_YYYYMMDDhhmmss_description.out` contains the auto-generated PR description. You can push it to the PR by running: + +```sh +gh pr edit -F relocate_YYYYMMDDhhmmss_description.out -R elastic/kibana +``` diff --git a/packages/kbn-relocate/constants.ts b/packages/kbn-relocate/constants.ts new file mode 100644 index 0000000000000..0ba7e9d50314b --- /dev/null +++ b/packages/kbn-relocate/constants.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import dedent from 'dedent'; + +export const BASE_FOLDER = process.cwd() + '/'; +export const BASE_FOLDER_DEPTH = process.cwd().split('/').length; +export const KIBANA_FOLDER = process.cwd().split('/').pop()!; +export const EXCLUDED_MODULES = ['@kbn/core']; +export const TARGET_FOLDERS = [ + 'src/platform/plugins/', + 'src/platform/packages/', + 'x-pack/platform/plugins/', + 'x-pack/platform/packages/', + 'x-pack/solutions/', +]; +export const EXTENSIONS = [ + 'eslintignore', + 'gitignore', + 'js', + 'mjs', + 'txt', + 'json', + 'lock', + 'bazel', + 'md', + 'mdz', + 'asciidoc', + 'sh', + 'ts', + 'jsonc', + 'yaml', + 'yml', +]; + +export const EXCLUDED_FOLDERS = [ + './api_docs', // autogenerated daily https://buildkite.com/elastic/kibana-api-docs-daily + './.chromium', + './.devcontainer', + './.es', + './.git', + // './.github', + './.native_modules', + './.node_binaries', + './.vscode', + './.yarn-local-mirror', + './build', + './core_http.codeql', + './data', + './node_modules', + './target', + './test.codeql', + './test2.codeql', + './trash', +]; + +export const NO_GREP = EXCLUDED_FOLDERS.map((f) => `--exclude-dir "${f}"`).join(' '); + +// These two constants are singletons, used and updated throughout the process +export const UPDATED_REFERENCES = new Set(); +export const UPDATED_RELATIVE_PATHS = new Set(); +export const SCRIPT_ERRORS: string[] = []; + +export const YMDMS = new Date() + .toISOString() + .replace(/[^0-9]/g, '') + .slice(0, -3); + +export const DESCRIPTION = `relocate_${YMDMS}_description.out`; +export const NEW_BRANCH = `kbn-team-1309-relocate-${YMDMS}`; + +export const GLOBAL_DESCRIPTION = dedent` +## Summary + +This PR aims at relocating some of the Kibana modules (plugins and packages) into a new folder structure, according to the _Sustainable Kibana Architecture_ initiative. + +> [!IMPORTANT] +> * We kindly ask you to: +> * Manually fix the errors in the error section below (if there are any). +> * Search for the \`packages[\/\\]\` and \`plugins[\/\\]\` patterns in the source code (Babel and Eslint config files), and update them appropriately. +> * Manually review \`.buildkite/scripts/pipelines/pull_request/pipeline.ts\` to ensure that any CI pipeline customizations continue to be correctly applied after the changed path names +> * Review all of the updated files, specially the \`.ts\` and \`.js\` files listed in the sections below, as some of them contain relative paths that have been updated. +> * Think of potential impact of the move, including tooling and configuration files that can be pointing to the relocated modules. E.g.: +> * customised eslint rules +> * docs pointing to source code + +> [!NOTE] +> * This PR has been auto-generated. +> * Any manual contributions will be lost if the 'relocate' script is re-run. +> * Try to obtain the missing reviews / approvals before applying manual fixes, and/or keep your changes in a .patch / git stash. +> * Please use [#sustainable_kibana_architecture](https://elastic.slack.com/archives/C07TCKTA22E) Slack channel for feedback. + +`; diff --git a/packages/kbn-relocate/index.ts b/packages/kbn-relocate/index.ts new file mode 100644 index 0000000000000..6270090533150 --- /dev/null +++ b/packages/kbn-relocate/index.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { run } from '@kbn/dev-cli-runner'; +import { findAndRelocateModules } from './relocate'; + +const toStringArray = (flag: string | boolean | string[] | undefined): string[] => { + if (typeof flag === 'string') { + return [flag].filter(Boolean); + } else if (typeof flag === 'boolean') { + return []; + } else if (Array.isArray(flag)) { + return flag.filter(Boolean); + } + return []; +}; + +const toOptString = ( + flagName: string, + flag: string | boolean | string[] | undefined, + defaultValue?: string +): string | undefined => { + if (typeof flag === 'boolean') { + throw Error(`You must specify a valid string for the --${flagName} flag`); + } else if (Array.isArray(flag)) { + throw Error(`Cannot specify multiple values for --${flagName} flag`); + } + return flag || defaultValue; +}; + +/** + * A CLI to move Kibana modules into the right folder structure, + * according to the Sustainable Kibana Architecture + */ +export const runKbnRelocateCli = () => { + run( + async ({ log, flags }) => { + const { pr, team, path, include, exclude, baseBranch } = flags; + await findAndRelocateModules({ + prNumber: toOptString('prNumber', pr), + baseBranch: toOptString('baseBranch', baseBranch, 'main')!, + teams: toStringArray(team), + paths: toStringArray(path), + included: toStringArray(include), + excluded: toStringArray(exclude), + log, + }); + }, + { + log: { + defaultLevel: 'info', + }, + flags: { + string: ['pr', 'team', 'path', 'include', 'exclude', 'baseBranch'], + help: ` + Usage: node scripts/relocate [options] + + --pr Use the given PR number instead of creating a new one + --team Include all modules (packages and plugins) belonging to the specified owner (can specify multiple teams) + --path Include all modules (packages and plugins) under the specified path (can specify multiple paths) + --include Include the specified module in the relocation (can specify multiple modules) + --exclude Exclude the specified module from the relocation (can use multiple times) + --baseBranch Use a branch different than 'main' (e.g. "8.x") + + E.g. relocate all modules owned by Core team and also modules owned by Operations team, excluding 'foo-module-id'. Force push into PR 239847: + node scripts/relocate --pr 239847 --team @elastic/kibana-core --team @elastic/kibana-operations --exclude @kbn/foo-module-id + `, + }, + } + ); +}; diff --git a/packages/kbn-relocate/jest.config.js b/packages/kbn-relocate/jest.config.js new file mode 100644 index 0000000000000..f63ccebc0ab46 --- /dev/null +++ b/packages/kbn-relocate/jest.config.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-relocate'], +}; diff --git a/packages/kbn-relocate/kibana.jsonc b/packages/kbn-relocate/kibana.jsonc new file mode 100644 index 0000000000000..bae4e5f6e9c60 --- /dev/null +++ b/packages/kbn-relocate/kibana.jsonc @@ -0,0 +1,6 @@ +{ + "type": "shared-server", + "id": "@kbn/relocate", + "owner": "@elastic/kibana-core", + "devOnly": true +} diff --git a/packages/kbn-relocate/package.json b/packages/kbn-relocate/package.json new file mode 100644 index 0000000000000..5091a883ce6a5 --- /dev/null +++ b/packages/kbn-relocate/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/relocate", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0" +} diff --git a/packages/kbn-relocate/relocate.ts b/packages/kbn-relocate/relocate.ts new file mode 100644 index 0000000000000..abd4eeced0855 --- /dev/null +++ b/packages/kbn-relocate/relocate.ts @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { join } from 'path'; +import { existsSync } from 'fs'; +import { rename, mkdir, rm } from 'fs/promises'; +import inquirer from 'inquirer'; +import { orderBy } from 'lodash'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { getPackages } from '@kbn/repo-packages'; +import { REPO_ROOT } from '@kbn/repo-info'; +import type { Package } from './types'; +import { + DESCRIPTION, + EXCLUDED_MODULES, + KIBANA_FOLDER, + NEW_BRANCH, + TARGET_FOLDERS, +} from './constants'; +import { + belongsTo, + calculateModuleTargetFolder, + replaceReferences, + replaceRelativePaths, +} from './utils.relocate'; +import { safeExec } from './utils.exec'; +import { relocatePlan, relocateSummary } from './utils.logging'; +import { checkoutBranch, checkoutResetPr } from './utils.git'; + +const relocateModule = async (module: Package, log: ToolingLog) => { + const destination = calculateModuleTargetFolder(module); + log.info(`Moving ${module.directory} to ${destination}`); + const chunks = destination.split('/'); + chunks.pop(); // discard module folder + if (existsSync(destination)) { + await rm(destination, { recursive: true }); + } + await mkdir(join('/', ...chunks), { recursive: true }); + await rename(module.directory, destination); + await replaceReferences(module, destination, log); + await replaceRelativePaths(module, destination, log); +}; + +const relocateModules = async (toMove: Package[], log: ToolingLog): Promise => { + let relocated: number = 0; + for (let i = 0; i < toMove.length; ++i) { + const module = toMove[i]; + + if (TARGET_FOLDERS.some((folder) => module.directory.includes(folder))) { + log.warning(`The module ${module.id} is already in a "sustainable" folder. Skipping`); + // skip modules that are already moved + continue; + } + log.info(''); + log.info('--------------------------------------------------------------------------------'); + log.info(`\t${module.id} (${i + 1} of ${toMove.length})`); + log.info('--------------------------------------------------------------------------------'); + await relocateModule(module, log); + + // after move operations + await safeExec('yarn kbn bootstrap'); + await safeExec('node scripts/build_plugin_list_docs'); + await safeExec('node scripts/generate codeowners'); + await safeExec('node scripts/lint_packages --fix'); + await safeExec('node scripts/eslint --no-cache --fix'); + await safeExec('node scripts/precommit_hook --fix'); + + // single commit per module now + await safeExec(`git add .`); + await safeExec(`git commit -m "Relocating module \\\`${module.id}\\\`"`); + ++relocated; + } + return relocated; +}; + +interface FindModulesParams { + teams: string[]; + paths: string[]; + included: string[]; + excluded: string[]; +} + +export interface RelocateModulesParams { + baseBranch: string; + prNumber?: string; + teams: string[]; + paths: string[]; + included: string[]; + excluded: string[]; + log: ToolingLog; +} + +const findModules = ({ teams, paths, included, excluded }: FindModulesParams) => { + // get all modules + const modules = getPackages(REPO_ROOT); + + // find modules selected by user filters + return orderBy( + modules + // exclude devOnly modules (they will remain in /packages) + .filter(({ manifest }) => !manifest.devOnly) + // exclude modules that do not specify a group + .filter(({ manifest }) => manifest.group) + // explicit exclusions + .filter(({ id }) => !EXCLUDED_MODULES.includes(id) && !excluded.includes(id)) + // we don't want to move test modules (just yet) + .filter( + ({ directory }) => + !directory.includes(`/${KIBANA_FOLDER}/test/`) && + !directory.includes(`/${KIBANA_FOLDER}/x-pack/test/`) + ) + // the module is under the umbrella specified by the user + .filter( + (module) => + included.includes(module.id) || + teams.some((team) => belongsTo(module, team)) || + paths.some((path) => module.directory.includes(path)) + ) + // the module is not explicitly excluded + .filter(({ id }) => !excluded.includes(id)), + 'id' + ); +}; + +export const findAndRelocateModules = async (params: RelocateModulesParams) => { + const { prNumber, log, baseBranch, ...findParams } = params; + + const toMove = findModules(findParams); + if (!toMove.length) { + log.info( + `No packages match the specified filters. Please tune your '--path' and/or '--team' and/or '--include' flags` + ); + return; + } + + relocatePlan(toMove, log); + const res1 = await inquirer.prompt({ + type: 'confirm', + name: 'confirmPlan', + message: `The script will RESET CHANGES in this repository, relocate the modules above and update references. Proceed?`, + }); + + if (!res1.confirmPlan) { + log.info('Aborting'); + return; + } + + // start with a clean repo + await safeExec(`git restore --staged .`); + await safeExec(`git restore .`); + await safeExec(`git clean -f -d`); + await safeExec(`git checkout ${baseBranch} && git pull upstream ${baseBranch} && git push`); + + if (prNumber) { + // checkout existing PR, reset all commits, rebase from baseBranch + try { + if (!(await checkoutResetPr(baseBranch, prNumber))) { + log.info('Aborting'); + return; + } + } catch (error) { + log.error(`Error checking out / resetting PR #${prNumber}:`); + log.error(error); + return; + } + } else { + // checkout [new] branch + await checkoutBranch(NEW_BRANCH); + } + + // relocate modules + await safeExec(`yarn kbn bootstrap`); + const movedCount = await relocateModules(toMove, log); + + if (movedCount === 0) { + log.warning( + 'No modules were relocated, aborting operation to prevent force-pushing empty changes (this would close the existing PR!)' + ); + return; + } + relocateSummary(log); + + // push changes in the branch + const res2 = await inquirer.prompt({ + type: 'confirm', + name: 'pushBranch', + message: `Relocation finished! You can commit extra changes at this point. Confirm to proceed pushing the current branch`, + }); + + const pushCmd = prNumber + ? `git push --force-with-lease` + : `git push --set-upstream origin ${NEW_BRANCH}`; + + if (!res2.pushBranch) { + log.info(`Remember to push changes with "${pushCmd}"`); + return; + } + await safeExec(pushCmd); + + if (prNumber) { + await safeExec(`gh pr edit ${prNumber} -F ${DESCRIPTION} -R elastic/kibana`); + log.info(`Access the PR at: https://github.com/elastic/kibana/pull/${prNumber}`); + } else { + log.info('TIP: Run the following command to quickly create a PR:'); + log.info(`$ gh pr create -d -t "" -F ${DESCRIPTION} -R elastic/kibana`); + } +}; diff --git a/packages/kbn-relocate/transforms.ts b/packages/kbn-relocate/transforms.ts new file mode 100644 index 0000000000000..72f57a24daa00 --- /dev/null +++ b/packages/kbn-relocate/transforms.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { Package } from './types'; + +type TransformFunction = (param: string) => string; +const TRANSFORMS: Record<string, string | TransformFunction> = { + 'x-pack/solutions/security/packages/security-solution/': 'x-pack/solutions/security/packages/', + 'x-pack/solutions/observability/plugins/observability_solution/': + 'x-pack/solutions/observability/plugins/', + 'x-pack/solutions/observability/packages/observability/': + 'x-pack/solutions/observability/packages/', + 'src/core/packages/core/': (path: string) => { + const relativePath = path.split('src/core/packages/')[1]; + const relativeChunks = relativePath.split('/'); + const packageName = relativeChunks.pop(); + const unneededPrefix = relativeChunks.join('-') + '-'; + + // strip the spare /core/ folder + path = path.replace('src/core/packages/core/', 'src/core/packages/'); + + if (packageName?.startsWith(unneededPrefix)) { + return path.replace(unneededPrefix, ''); + } else { + return path; + } + }, +}; +export const applyTransforms = (module: Package, path: string): string => { + const transform = Object.entries(TRANSFORMS).find(([what]) => path.includes(what)); + if (!transform) { + return path; + } else { + const [what, by] = transform; + if (typeof by === 'function') { + return by(path); + } else if (typeof by === 'string') { + return path.replace(what, by); + } else { + throw new Error('Invalid transform function', by); + } + } +}; diff --git a/packages/kbn-relocate/tsconfig.json b/packages/kbn-relocate/tsconfig.json new file mode 100644 index 0000000000000..1ee41aafca1ee --- /dev/null +++ b/packages/kbn-relocate/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/dev-cli-runner", + "@kbn/repo-info", + "@kbn/repo-packages", + "@kbn/safer-lodash-set", + "@kbn/tooling-log", + ] +} diff --git a/packages/kbn-relocate/types.ts b/packages/kbn-relocate/types.ts new file mode 100644 index 0000000000000..3826561f523e1 --- /dev/null +++ b/packages/kbn-relocate/types.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export type { Package } from '@kbn/repo-packages'; + +export interface CommitAuthor { + login: string; +} + +export interface Commit { + messageHeadline: string; + authors: CommitAuthor[]; +} + +export interface PullRequest { + number: string; + commits: Commit[]; +} diff --git a/packages/kbn-relocate/utils.exec.ts b/packages/kbn-relocate/utils.exec.ts new file mode 100644 index 0000000000000..afd9391dfcab5 --- /dev/null +++ b/packages/kbn-relocate/utils.exec.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import util from 'util'; +import { exec } from 'child_process'; + +export const execAsync = util.promisify(exec); + +export const safeExec = async (command: string, critical = true, log = true) => { + try { + if (log) { + // eslint-disable-next-line no-console + console.log(' >', command); + } + const result = await execAsync(command, { maxBuffer: 1024 * 1024 * 128 }); + return result; + } catch (err) { + const message = `Error executing ${command}: ${err}`; + + if (critical) { + throw err; + } + return { stdout: '', stderr: message }; + } +}; + +export const quietExec = async (command: string) => { + return await safeExec(command, false, false); +}; diff --git a/packages/kbn-relocate/utils.git.ts b/packages/kbn-relocate/utils.git.ts new file mode 100644 index 0000000000000..7f3386bdc32f1 --- /dev/null +++ b/packages/kbn-relocate/utils.git.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import inquirer from 'inquirer'; +import type { Commit, PullRequest } from './types'; +import { safeExec } from './utils.exec'; + +export const findPr = async (number: string): Promise<PullRequest> => { + const commits = JSON.parse( + (await safeExec(`gh pr view ${number} --json commits`)).stdout + ).commits; + return { number, commits }; +}; + +export function hasManualCommits(commits: Commit[]) { + const manualCommits = commits.filter( + (commit) => + !commit.messageHeadline.startsWith('Relocating module ') && + !commit.messageHeadline.startsWith('Moving modules owned by ') && + commit.authors.some((author) => author.login !== 'kibanamachine') + ); + + return manualCommits.length > 0; +} + +export async function getLastCommitMessage() { + return (await safeExec('git log -1 --pretty=%B')).stdout.split('\n')[0]; +} + +export async function resetAllCommits(numCommits: number) { + await safeExec(`git reset --hard HEAD~${numCommits}`); + + let msg = await getLastCommitMessage(); + while (msg.startsWith('Relocating module ')) { + await safeExec(`git reset --hard HEAD~1`); + msg = await getLastCommitMessage(); + } + await safeExec('git restore --staged .'); + await safeExec('git restore .'); + await safeExec('git clean -f -d'); +} + +export async function localBranchExists(branchName: string): Promise<boolean> { + const res = await safeExec('git branch -l'); + const branches = res.stdout + .split('\n') + .filter(Boolean) + .map((name) => name.trim()); + return branches.includes(branchName); +} + +export const checkoutResetPr = async (baseBranch: string, prNumber: string): Promise<boolean> => { + const pr = await findPr(prNumber); + + if (hasManualCommits(pr.commits)) { + const res = await inquirer.prompt({ + type: 'confirm', + name: 'overrideManualCommits', + message: 'Detected manual commits in the PR, do you want to override them?', + }); + if (!res.overrideManualCommits) { + return false; + } + } + + // previous cleanup TODO REMOVE + await safeExec(`git restore --staged .`); + await safeExec(`git restore .`); + await safeExec(`git clean -f -d`); + + // checkout the PR branch + await safeExec(`gh pr checkout ${prNumber}`); + await resetAllCommits(pr.commits.length); + await safeExec(`git rebase ${baseBranch}`); + return true; +}; + +export const checkoutBranch = async (branch: string) => { + // create a new branch / PR + if (await localBranchExists(branch)) { + throw new Error('The local branch already exists, aborting!'); + } else { + await safeExec(`git checkout -b ${branch}`); + } +}; diff --git a/packages/kbn-relocate/utils.logging.ts b/packages/kbn-relocate/utils.logging.ts new file mode 100644 index 0000000000000..5b290292dd75e --- /dev/null +++ b/packages/kbn-relocate/utils.logging.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { ToolingLog } from '@kbn/tooling-log'; +import { appendFileSync, writeFileSync } from 'fs'; +import dedent from 'dedent'; +import type { Package } from './types'; +import { calculateModuleTargetFolder } from './utils.relocate'; +import { + BASE_FOLDER, + DESCRIPTION, + GLOBAL_DESCRIPTION, + SCRIPT_ERRORS, + UPDATED_REFERENCES, + UPDATED_RELATIVE_PATHS, +} from './constants'; + +export const relocatePlan = (modules: Package[], log: ToolingLog) => { + const plugins = modules.filter((module) => module.manifest.type === 'plugin'); + const packages = modules.filter((module) => module.manifest.type !== 'plugin'); + + const target = (module: Package) => calculateModuleTargetFolder(module).replace(BASE_FOLDER, ''); + writeFileSync(DESCRIPTION, GLOBAL_DESCRIPTION); + + if (plugins.length) { + const pluginList = dedent` + \n\n#### ${plugins.length} plugin(s) are going to be relocated:\n + | Id | Target folder | + | -- | ------------- | + ${plugins.map((plg) => `| \`${plg.id}\` | \`${target(plg)}\` |`).join('\n')} + \n\n`; + + appendFileSync(DESCRIPTION, pluginList); + log.info( + `${plugins.length} plugin(s) are going to be relocated:\n${plugins + .map((plg) => `${plg.id} => ${target(plg)}`) + .join('\n')}` + ); + } + + if (packages.length) { + const packageList = dedent` + \n\n#### ${packages.length} packages(s) are going to be relocated:\n + | Id | Target folder | + | -- | ------------- | + ${packages.map((pkg) => `| \`${pkg.id}\` | \`${target(pkg)}\` |`).join('\n')} + \n\n`; + + appendFileSync(DESCRIPTION, packageList); + log.info( + `${packages.length} packages(s) are going to be relocated:\n${packages + .map((plg) => `${plg.id} => ${target(plg)}`) + .join('\n')}` + ); + } +}; + +export const appendCollapsible = ( + fileName: string, + title: string, + contents: string, + open = false +) => { + appendFileSync( + fileName, + dedent` + <details ${open ? 'open' : ''}> + <summary>${title}</summary> + + \`\`\` + ${contents} + \`\`\` + + </details>` + ); +}; + +export const relocateSummary = (log: ToolingLog) => { + if (SCRIPT_ERRORS.length > 0) { + const contents = SCRIPT_ERRORS.sort().join('\n'); + appendCollapsible(DESCRIPTION, 'Script errors', contents, true); + log.warning(`Please address the following errors:\n${contents}`); + } + + if (UPDATED_REFERENCES.size > 0) { + const contents = Array.from(UPDATED_REFERENCES).sort().join('\n'); + appendCollapsible(DESCRIPTION, 'Updated references', contents); + log.info( + `The following files have been updated to replace references to modules:\n${contents}` + ); + } + + if (UPDATED_RELATIVE_PATHS.size > 0) { + const contents = Array.from(UPDATED_RELATIVE_PATHS) + .sort() + .map((ref) => ref.replace(BASE_FOLDER, '')) + .join('\n'); + appendCollapsible(DESCRIPTION, 'Updated relative paths', contents); + log.info(`The following files contain relative paths that have been updated:\n${contents}`); + } +}; diff --git a/packages/kbn-relocate/utils.relocate.ts b/packages/kbn-relocate/utils.relocate.ts new file mode 100644 index 0000000000000..c76c1f48790ba --- /dev/null +++ b/packages/kbn-relocate/utils.relocate.ts @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { join } from 'path'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { orderBy } from 'lodash'; +import type { Package } from './types'; +import { applyTransforms } from './transforms'; +import { + BASE_FOLDER, + BASE_FOLDER_DEPTH, + EXTENSIONS, + KIBANA_FOLDER, + NO_GREP, + SCRIPT_ERRORS, + TARGET_FOLDERS, + UPDATED_REFERENCES, + UPDATED_RELATIVE_PATHS, +} from './constants'; +import { quietExec, safeExec } from './utils.exec'; + +export const belongsTo = (module: Package, owner: string): boolean => { + return Array.from(module.manifest.owner)[0] === owner; +}; + +export const stripFirstChunk = (path: string): string => { + const chunks = path.split('/'); + chunks.shift(); + return chunks.join('/'); +}; + +export const calculateModuleTargetFolder = (module: Package): string => { + const group = module.manifest.group!; + const isPlugin = module.manifest.type === 'plugin'; + const fullPath = join(BASE_FOLDER, module.directory); + let moduleDelimiter = isPlugin ? '/plugins/' : '/packages/'; + if (TARGET_FOLDERS.some((folder) => module.directory.includes(folder)) && group === 'platform') { + // if a platform module has already been relocated, strip the /private/ or /shared/ part too + moduleDelimiter += `${module.visibility}/`; + } + const moduleFolder = fullPath.split(moduleDelimiter).pop()!; + let path: string; + + if (group === 'platform') { + if (fullPath.includes(`/${KIBANA_FOLDER}/packages/core/`)) { + // packages/core/* => src/core/packages/* + path = join(BASE_FOLDER, 'src', 'core', 'packages', moduleFolder); + } else { + const isXpack = fullPath.includes(`/${KIBANA_FOLDER}/x-pack/`); + const visibility = module.manifest.visibility!; + + path = join( + BASE_FOLDER, + isXpack ? 'x-pack' : 'src', + group, + isPlugin ? 'plugins' : 'packages', + visibility, + moduleFolder + ); + } + } else { + path = join( + BASE_FOLDER, + 'x-pack', // all solution modules are 'x-pack' + 'solutions', + group, + isPlugin ? 'plugins' : 'packages', + moduleFolder + ); + } + + // after-creation transforms + return applyTransforms(module, path); +}; + +export const replaceReferences = async (module: Package, destination: string, log: ToolingLog) => { + const dir = module.directory; + const source = + dir.startsWith(KIBANA_FOLDER) || dir.startsWith(`/${KIBANA_FOLDER}`) + ? join(BASE_FOLDER, dir) + : dir; + const relativeSource = source.replace(BASE_FOLDER, ''); + const relativeDestination = destination.replace(BASE_FOLDER, ''); + + if ( + (relativeSource.startsWith('src') && relativeDestination.startsWith('src')) || + (relativeSource.startsWith('x-pack') && relativeDestination.startsWith('x-pack')) + ) { + await replaceReferencesInternal( + stripFirstChunk(relativeSource), + stripFirstChunk(relativeDestination), + log + ); + } else { + await replaceReferencesInternal(relativeSource, relativeDestination, log); + } +}; + +const replaceReferencesInternal = async ( + relativeSource: string, + relativeDestination: string, + log: ToolingLog +) => { + log.info(`Finding and replacing "${relativeSource}" by "${relativeDestination}"`); + + const src = relativeSource.replaceAll('/', '\\/'); + const dst = relativeDestination.replaceAll('/', '\\/'); + + const result = await safeExec( + `grep -I -s -R -l ${EXTENSIONS.map((ext) => `--include="*.${ext}"`).join(' ')} \ + ${NO_GREP} "${relativeSource}"`, + false + ); + + const matchingFiles = result.stdout.split('\n').filter(Boolean); + + for (let i = 0; i < matchingFiles.length; ++i) { + const file = matchingFiles[i]; + if (file.includes('/target/types/') || file.includes('/target/public/')) { + continue; + } + + const md5Before = (await quietExec(`md5 ${file} --quiet`)).stdout.trim(); + // if we are updating packages/cloud references, we must pay attention to not update packages/cloud_defend too + await safeExec(`sed -i '' -E "/${src}[\-_a-zA-Z0-9]/! s/${src}/${dst}/g" ${file}`, false); + const md5After = (await quietExec(`md5 ${file} --quiet`)).stdout.trim(); + + if (md5Before !== md5After) { + UPDATED_REFERENCES.add(file); + } + } + + // plugins\/pluginName special treatment (.buildkite/scripts/pipelines/pull_request/pipeline.ts) + const backFwdSrc = relativeSource.replaceAll('/', `\\\\\\/`); + const backFwdDst = relativeDestination.replaceAll('/', `\\\\\\/`); + await safeExec( + `sed -i '' -E '/${src}[\-_a-zA-Z0-9]/! s/${backFwdSrc}/${backFwdDst}/g' .buildkite/scripts/pipelines/pull_request/pipeline.ts`, + false + ); +}; + +const getRelativeDepth = (directory: string): number => { + const fullPath = directory.startsWith(BASE_FOLDER) ? directory : join(BASE_FOLDER, directory); + return fullPath.split('/').length - BASE_FOLDER_DEPTH; +}; + +export const replaceRelativePaths = async ( + module: Package, + destination: string, + log: ToolingLog +) => { + log.info('Updating relative paths at fault'); + + const relativeDepthBefore = getRelativeDepth(module.directory); + const relativeDepthAfter = getRelativeDepth(destination); + const relativeDepthDiff = relativeDepthAfter - relativeDepthBefore; + + const result = await safeExec( + `grep -I -s -R -n -o ${NO_GREP} -E "\\.\\.(/\\.\\.)+/?" ${destination}`, + false + ); + const matches = result.stdout.split('\n').filter(Boolean); + + const brokenReferences = orderBy( + matches + .map((line) => line.split(':')) + .map(([path, line, match]) => { + if (match.endsWith('/')) { + match = match.substring(0, match.length - 1); + } + let moduleRelativePath = path.replace(destination, ''); + if (moduleRelativePath.startsWith('/')) { + moduleRelativePath = moduleRelativePath.substring(1); + } + const moduleRelativeDepth = moduleRelativePath.split('/').length - 1; // do not count filename + const matchDepth = match.split('/').length; + + return { path, line, moduleRelativeDepth, match, matchDepth }; + }) + .filter(({ matchDepth, moduleRelativeDepth }) => matchDepth > moduleRelativeDepth), + 'matchDepth', + 'desc' + ); + + for (let i = 0; i < brokenReferences.length; ++i) { + const { path, line, match, matchDepth } = brokenReferences[i]; + + if (path.includes('/target/types/') || path.includes('/target/public/')) { + continue; + } + const pathLine = `${path}:${line}`; + + if (UPDATED_RELATIVE_PATHS.has(pathLine)) { + const message = `Cannot replace multiple occurrences of "${match}" in the same line, please fix manually:\t${pathLine}`; + SCRIPT_ERRORS.push(message); + } else { + const escapedMatch = match.replaceAll('/', '\\/').replaceAll('.', '\\.'); // escape '.' too (regexp any char) + const escapedReplacement = new Array(matchDepth + relativeDepthDiff).fill('..').join('\\/'); + + await safeExec(`sed -i '' "${line}s/${escapedMatch}/${escapedReplacement}/" ${path}`, false); + UPDATED_RELATIVE_PATHS.add(pathLine); + } + } +}; diff --git a/scripts/relocate.js b/scripts/relocate.js new file mode 100644 index 0000000000000..1a8c71373c673 --- /dev/null +++ b/scripts/relocate.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +require('../src/setup_node_env'); +require('@kbn/relocate').runKbnRelocateCli(); diff --git a/tsconfig.base.json b/tsconfig.base.json index 1d7528bfd3e14..d083eb3ad4748 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1440,6 +1440,8 @@ "@kbn/react-mute-legacy-root-warning/*": ["packages/kbn-react-mute-legacy-root-warning/*"], "@kbn/recently-accessed": ["packages/kbn-recently-accessed"], "@kbn/recently-accessed/*": ["packages/kbn-recently-accessed/*"], + "@kbn/relocate": ["packages/kbn-relocate"], + "@kbn/relocate/*": ["packages/kbn-relocate/*"], "@kbn/remote-clusters-plugin": ["x-pack/plugins/remote_clusters"], "@kbn/remote-clusters-plugin/*": ["x-pack/plugins/remote_clusters/*"], "@kbn/rendering-plugin": ["test/plugin_functional/plugins/rendering_plugin"], diff --git a/yarn.lock b/yarn.lock index 93cc85d36cf16..f9660a9b59220 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6696,6 +6696,10 @@ version "0.0.0" uid "" +"@kbn/relocate@link:packages/kbn-relocate": + version "0.0.0" + uid "" + "@kbn/remote-clusters-plugin@link:x-pack/plugins/remote_clusters": version "0.0.0" uid "" From 2523675800fa6d6c67e2db0d555c212dcb20a4af Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:21:57 +0000 Subject: [PATCH 2/4] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index df1caa990abd4..2952f4ec53f1f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -434,6 +434,7 @@ packages/kbn-react-field @elastic/kibana-data-discovery packages/kbn-react-hooks @elastic/obs-ux-logs-team packages/kbn-react-mute-legacy-root-warning @elastic/appex-sharedux packages/kbn-recently-accessed @elastic/appex-sharedux +packages/kbn-relocate @elastic/kibana-core packages/kbn-repo-file-maps @elastic/kibana-operations packages/kbn-repo-info @elastic/kibana-operations packages/kbn-repo-linter @elastic/kibana-operations From 623038a2c67ddcab992243074dc25b2cdc911269 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:33:23 +0000 Subject: [PATCH 3/4] [CI] Auto-commit changed files from 'node scripts/notice' --- packages/kbn-relocate/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/kbn-relocate/tsconfig.json b/packages/kbn-relocate/tsconfig.json index 1ee41aafca1ee..cd11268486134 100644 --- a/packages/kbn-relocate/tsconfig.json +++ b/packages/kbn-relocate/tsconfig.json @@ -17,7 +17,6 @@ "@kbn/dev-cli-runner", "@kbn/repo-info", "@kbn/repo-packages", - "@kbn/safer-lodash-set", "@kbn/tooling-log", ] } From 6a6554112c1c8163a8c03de685191142779c7665 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila <gerard.soldevila@elastic.co> Date: Wed, 11 Dec 2024 15:41:44 +0100 Subject: [PATCH 4/4] Allow to simply move an individual module --- packages/kbn-relocate/index.ts | 30 ++++++++++++++++++------------ packages/kbn-relocate/relocate.ts | 13 +++++++++++-- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/kbn-relocate/index.ts b/packages/kbn-relocate/index.ts index 6270090533150..cddaa307ab7b6 100644 --- a/packages/kbn-relocate/index.ts +++ b/packages/kbn-relocate/index.ts @@ -8,7 +8,7 @@ */ import { run } from '@kbn/dev-cli-runner'; -import { findAndRelocateModules } from './relocate'; +import { findAndRelocateModules, findAndMoveModule } from './relocate'; const toStringArray = (flag: string | boolean | string[] | undefined): string[] => { if (typeof flag === 'string') { @@ -41,26 +41,32 @@ const toOptString = ( export const runKbnRelocateCli = () => { run( async ({ log, flags }) => { - const { pr, team, path, include, exclude, baseBranch } = flags; - await findAndRelocateModules({ - prNumber: toOptString('prNumber', pr), - baseBranch: toOptString('baseBranch', baseBranch, 'main')!, - teams: toStringArray(team), - paths: toStringArray(path), - included: toStringArray(include), - excluded: toStringArray(exclude), - log, - }); + if (typeof flags.moveOnly === 'string' && flags.moveOnly.length > 0) { + log.info('When using --moveOnly flag, the rest of flags are ignored.'); + await findAndMoveModule(flags.moveOnly, log); + } else { + const { pr, team, path, include, exclude, baseBranch } = flags; + await findAndRelocateModules({ + prNumber: toOptString('prNumber', pr), + baseBranch: toOptString('baseBranch', baseBranch, 'main')!, + teams: toStringArray(team), + paths: toStringArray(path), + included: toStringArray(include), + excluded: toStringArray(exclude), + log, + }); + } }, { log: { defaultLevel: 'info', }, flags: { - string: ['pr', 'team', 'path', 'include', 'exclude', 'baseBranch'], + string: ['pr', 'team', 'path', 'include', 'exclude', 'baseBranch', 'moveOnly'], help: ` Usage: node scripts/relocate [options] + --moveOnly <moduleId> Only move the specified module in the current branch (no cleanup, no branching, no commit) --pr <number> Use the given PR number instead of creating a new one --team <owner> Include all modules (packages and plugins) belonging to the specified owner (can specify multiple teams) --path <path> Include all modules (packages and plugins) under the specified path (can specify multiple paths) diff --git a/packages/kbn-relocate/relocate.ts b/packages/kbn-relocate/relocate.ts index abd4eeced0855..57d2d37b87ec7 100644 --- a/packages/kbn-relocate/relocate.ts +++ b/packages/kbn-relocate/relocate.ts @@ -33,7 +33,7 @@ import { safeExec } from './utils.exec'; import { relocatePlan, relocateSummary } from './utils.logging'; import { checkoutBranch, checkoutResetPr } from './utils.git'; -const relocateModule = async (module: Package, log: ToolingLog) => { +const moveModule = async (module: Package, log: ToolingLog) => { const destination = calculateModuleTargetFolder(module); log.info(`Moving ${module.directory} to ${destination}`); const chunks = destination.split('/'); @@ -61,7 +61,7 @@ const relocateModules = async (toMove: Package[], log: ToolingLog): Promise<numb log.info('--------------------------------------------------------------------------------'); log.info(`\t${module.id} (${i + 1} of ${toMove.length})`); log.info('--------------------------------------------------------------------------------'); - await relocateModule(module, log); + await moveModule(module, log); // after move operations await safeExec('yarn kbn bootstrap'); @@ -128,6 +128,15 @@ const findModules = ({ teams, paths, included, excluded }: FindModulesParams) => ); }; +export const findAndMoveModule = async (moduleId: string, log: ToolingLog) => { + const modules = findModules({ teams: [], paths: [], included: [moduleId], excluded: [] }); + if (!modules.length) { + log.warning(`Cannot move ${moduleId}, either not found or not allowed!`); + } else { + await moveModule(modules[0], log); + } +}; + export const findAndRelocateModules = async (params: RelocateModulesParams) => { const { prNumber, log, baseBranch, ...findParams } = params;