Sustainable Kibana Architecture: Add scripts/relocate CLI (beta) (e…

…lastic#203803)

## Summary

Adds the `node scripts/relocate` functionality, that helps moving
modules to their intended locations, according to the _Sustainable
Kibana Architecture_.

Please refer to the README.md for further details and usage.

---------

Co-authored-by: kibanamachine <[email protected]>

.github/CODEOWNERS

@@ -434,6 +434,7 @@ packages/kbn-react-field @elastic/kibana-data-discovery
 packages/kbn-react-hooks @elastic/obs-ux-logs-team
 packages/kbn-react-mute-legacy-root-warning @elastic/appex-sharedux
 packages/kbn-recently-accessed @elastic/appex-sharedux
+packages/kbn-relocate @elastic/kibana-core
 packages/kbn-repo-file-maps @elastic/kibana-operations
 packages/kbn-repo-info @elastic/kibana-operations
 packages/kbn-repo-linter @elastic/kibana-operations

package.json

@@ -1490,6 +1490,7 @@
     "@kbn/plugin-generator": "link:packages/kbn-plugin-generator",
     "@kbn/plugin-helpers": "link:packages/kbn-plugin-helpers",
     "@kbn/product-doc-artifact-builder": "link:x-pack/packages/ai-infra/product-doc-artifact-builder",
+    "@kbn/relocate": "link:packages/kbn-relocate",
     "@kbn/repo-file-maps": "link:packages/kbn-repo-file-maps",
     "@kbn/repo-linter": "link:packages/kbn-repo-linter",
     "@kbn/repo-path": "link:packages/kbn-repo-path",

packages/kbn-relocate/README.md

@@ -0,0 +1,64 @@
+# @kbn/relocate
+
+This package contains a CLI tool to help move modules (plugins and packages) into their intended folders, according to the _Sustainable Kibana Architecture.
+
+## Prerequisites
+
+You must have `gh` CLI tool installed. You can install it by running:
+
+```sh
+brew install gh
+```
+
+You must also configure your "default" kibana repo in `gh`, so that it can find PRs.:
+
+```sh
+gh repo set-default elastic/kibana
+```
+
+You must have `elastic/kibana` remote configured under the name `upstream`.
+
+## Usage
+
+First of all, you need to decide whether you want to contribute to an existing PR or to create a new one. Use the `--pr` flag to specify the PR you are trying to update:
+
+```sh
+node scripts/relocate --pr <prNumber>
+```
+
+Note that when specifying an existing PR, the logic will undo + rewrite history for that PR, by force-pushing changes.
+
+To relocate modules for a given team, identify the "team handle" (e.g. @elastic/kibana-core), and run the following command from the root of the Kibana repo:
+
+```sh
+node scripts/relocate --pr <prNumber> --team <team_handle>
+```
+
+You can relocate modules by path, e.g. all modules that are under `x-pack/plugins/observability_solution/`:
+
+```sh
+node scripts/relocate --pr <prNumber> --path "x-pack/plugins/observability_solution/"
+```
+
+You can specify indivual packages by ID:
+
+```sh
+node scripts/relocate --pr <prNumber> --include "@kbn/data-forge" --include "@kbn/deeplinks-observability"
+```
+
+You can also specify combinations of the above filters, to include modules that match ANY of the criteria.
+Excluding modules explictly is also supported:
+
+```sh
+node scripts/relocate --pr <prNumber> --team "@elastic/obs-ux-management-team" --exclude "@kbn/data-forge"
+```
+
+## Details
+
+The script generates log / description files of the form `relocate_YYYYMMDDhhmmss_<type>.out`. You can inspect them if you encounter any errors.
+
+In particular, the file `relocate_YYYYMMDDhhmmss_description.out` contains the auto-generated PR description. You can push it to the PR by running:
+
+```sh
+gh pr edit <prNumber> -F relocate_YYYYMMDDhhmmss_description.out -R elastic/kibana
+```

packages/kbn-relocate/constants.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import dedent from 'dedent';
+
+export const BASE_FOLDER = process.cwd() + '/';
+export const BASE_FOLDER_DEPTH = process.cwd().split('/').length;
+export const KIBANA_FOLDER = process.cwd().split('/').pop()!;
+export const EXCLUDED_MODULES = ['@kbn/core'];
+export const TARGET_FOLDERS = [
+  'src/platform/plugins/',
+  'src/platform/packages/',
+  'x-pack/platform/plugins/',
+  'x-pack/platform/packages/',
+  'x-pack/solutions/',
+];
+export const EXTENSIONS = [
+  'eslintignore',
+  'gitignore',
+  'js',
+  'mjs',
+  'txt',
+  'json',
+  'lock',
+  'bazel',
+  'md',
+  'mdz',
+  'asciidoc',
+  'sh',
+  'ts',
+  'jsonc',
+  'yaml',
+  'yml',
+];
+
+export const EXCLUDED_FOLDERS = [
+  './api_docs', // autogenerated daily https://buildkite.com/elastic/kibana-api-docs-daily
+  './.chromium',
+  './.devcontainer',
+  './.es',
+  './.git',
+  // './.github',
+  './.native_modules',
+  './.node_binaries',
+  './.vscode',
+  './.yarn-local-mirror',
+  './build',
+  './core_http.codeql',
+  './data',
+  './node_modules',
+  './target',
+  './test.codeql',
+  './test2.codeql',
+  './trash',
+];
+
+export const NO_GREP = EXCLUDED_FOLDERS.map((f) => `--exclude-dir "${f}"`).join(' ');
+
+// These two constants are singletons, used and updated throughout the process
+export const UPDATED_REFERENCES = new Set<string>();
+export const UPDATED_RELATIVE_PATHS = new Set<string>();
+export const SCRIPT_ERRORS: string[] = [];
+
+export const YMDMS = new Date()
+  .toISOString()
+  .replace(/[^0-9]/g, '')
+  .slice(0, -3);
+
+export const DESCRIPTION = `relocate_${YMDMS}_description.out`;
+export const NEW_BRANCH = `kbn-team-1309-relocate-${YMDMS}`;
+
+export const GLOBAL_DESCRIPTION = dedent`
+## Summary
+
+This PR aims at relocating some of the Kibana modules (plugins and packages) into a new folder structure, according to the _Sustainable Kibana Architecture_ initiative.
+
+> [!IMPORTANT]
+> * We kindly ask you to:
+>   * Manually fix the errors in the error section below (if there are any).
+>   * Search for the \`packages[\/\\]\` and \`plugins[\/\\]\` patterns in the source code (Babel and Eslint config files), and update them appropriately.
+>   * Manually review \`.buildkite/scripts/pipelines/pull_request/pipeline.ts\` to ensure that any CI pipeline customizations continue to be correctly applied after the changed path names
+>   * Review all of the updated files, specially the \`.ts\` and \`.js\` files listed in the sections below, as some of them contain relative paths that have been updated.
+>   * Think of potential impact of the move, including tooling and configuration files that can be pointing to the relocated modules. E.g.:
+>     * customised eslint rules
+>     * docs pointing to source code
+
+> [!NOTE]
+> * This PR has been auto-generated.
+> * Any manual contributions will be lost if the 'relocate' script is re-run.
+> * Try to obtain the missing reviews / approvals before applying manual fixes, and/or keep your changes in a .patch / git stash.
+> * Please use [#sustainable_kibana_architecture](https://elastic.slack.com/archives/C07TCKTA22E) Slack channel for feedback.
+
+`;

packages/kbn-relocate/index.ts

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { run } from '@kbn/dev-cli-runner';
+import { findAndRelocateModules, findAndMoveModule } from './relocate';
+
+const toStringArray = (flag: string | boolean | string[] | undefined): string[] => {
+  if (typeof flag === 'string') {
+    return [flag].filter(Boolean);
+  } else if (typeof flag === 'boolean') {
+    return [];
+  } else if (Array.isArray(flag)) {
+    return flag.filter(Boolean);
+  }
+  return [];
+};
+
+const toOptString = (
+  flagName: string,
+  flag: string | boolean | string[] | undefined,
+  defaultValue?: string
+): string | undefined => {
+  if (typeof flag === 'boolean') {
+    throw Error(`You must specify a valid string for the --${flagName} flag`);
+  } else if (Array.isArray(flag)) {
+    throw Error(`Cannot specify multiple values for --${flagName} flag`);
+  }
+  return flag || defaultValue;
+};
+
+/**
+ * A CLI to move Kibana modules into the right folder structure,
+ * according to the Sustainable Kibana Architecture
+ */
+export const runKbnRelocateCli = () => {
+  run(
+    async ({ log, flags }) => {
+      if (typeof flags.moveOnly === 'string' && flags.moveOnly.length > 0) {
+        log.info('When using --moveOnly flag, the rest of flags are ignored.');
+        await findAndMoveModule(flags.moveOnly, log);
+      } else {
+        const { pr, team, path, include, exclude, baseBranch } = flags;
+        await findAndRelocateModules({
+          prNumber: toOptString('prNumber', pr),
+          baseBranch: toOptString('baseBranch', baseBranch, 'main')!,
+          teams: toStringArray(team),
+          paths: toStringArray(path),
+          included: toStringArray(include),
+          excluded: toStringArray(exclude),
+          log,
+        });
+      }
+    },
+    {
+      log: {
+        defaultLevel: 'info',
+      },
+      flags: {
+        string: ['pr', 'team', 'path', 'include', 'exclude', 'baseBranch', 'moveOnly'],
+        help: `
+          Usage: node scripts/relocate [options]
+
+          --moveOnly <moduleId> Only move the specified module in the current branch (no cleanup, no branching, no commit)
+          --pr <number> Use the given PR number instead of creating a new one
+          --team <owner> Include all modules (packages and plugins) belonging to the specified owner (can specify multiple teams)
+          --path <path> Include all modules (packages and plugins) under the specified path (can specify multiple paths)
+          --include <id> Include the specified module in the relocation (can specify multiple modules)
+          --exclude <id> Exclude the specified module from the relocation (can use multiple times)
+          --baseBranch <name> Use a branch different than 'main' (e.g. "8.x")
+
+          E.g. relocate all modules owned by Core team and also modules owned by Operations team, excluding 'foo-module-id'. Force push into PR 239847:
+          node scripts/relocate --pr 239847 --team @elastic/kibana-core --team @elastic/kibana-operations --exclude @kbn/foo-module-id
+        `,
+      },
+    }
+  );
+};

packages/kbn-relocate/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-relocate'],
+};

packages/kbn-relocate/kibana.jsonc

@@ -0,0 +1,6 @@
+{
+  "type": "shared-server",
+  "id": "@kbn/relocate",
+  "owner": "@elastic/kibana-core",
+  "devOnly": true
+}

packages/kbn-relocate/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/relocate",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}