feat(build-cli): New command transform:releaseNotes (#22466)

## The problem

Our goal is to automate the release process as much as possible. To that
end, now that we have a release notes generator, we'd like to
automatically upload those notes to the GitHub release as part of the
automated release.

Unfortunately, when we generate changelogs as part of release, we
consume and delete all the changesets for the release. However, we
generate a markdown file with the release notes as part of the release,
and that gets committed to the repo. Alas, we need the markdown to be
slightly different when posted to GitHub releases, and we don't have the
input changesets anymore, so we can't easily regenerate them in
automation (we'd need to find the commit in which the changesets were
consumed, go to the commit before, regenerate - it's not trivial).

## The proposed change

Since we have the release notes files in the repo, I have created a new
command, `transform:releaseNotes`, which takes the in-repo release notes
file as input, and outputs a modified markdown file for use in the
release pipeline. For example:

`flub transform releaseNotes --inFile RELEASE_NOTES/2.2.0.md --outFile
out.md`

---------

Co-authored-by: Alex Villarreal <[email protected]>
Co-authored-by: jzaffiro <[email protected]>

build-tools/packages/build-cli/README.md

@@ -48,6 +48,7 @@ USAGE
 * [`flub release`](docs/release.md) - Release commands are used to manage the Fluid release process.
 * [`flub rename-types`](docs/rename-types.md) - Renames type declaration files from .d.ts to .d.mts.
 * [`flub run`](docs/run.md) - Generate a report from input bundle stats collected through the collect bundleStats command.
+* [`flub transform`](docs/transform.md) - Transform commands are used to transform code, docs, etc. into alternative forms.
 * [`flub typetests`](docs/typetests.md) - Updates configuration for type tests in package.json files. If the previous version changes after running preparation, then npm install must be run before building.
 
 <!-- commandsstop -->

build-tools/packages/build-cli/docs/transform.md

@@ -0,0 +1,30 @@
+`flub transform`
+================
+
+Transform commands are used to transform code, docs, etc. into alternative forms.
+
+* [`flub transform releaseNotes`](#flub-transform-releasenotes)
+
+## `flub transform releaseNotes`
+
+Transforms a markdown release notes file into a format appropriate for use in a GitHub Release. This is used to transform in-repo release notes such that they can be automatically posted to our GitHub Releases.
+
+```
+USAGE
+  $ flub transform releaseNotes --inFile <value> --outFile <value> [-v | --quiet]
+
+FLAGS
+  --inFile=<value>   (required) A release notes file that was generated using 'flub generate releaseNotes'.
+  --outFile=<value>  (required) Output the transformed content to this file.
+
+LOGGING FLAGS
+  -v, --verbose  Enable verbose logging.
+      --quiet    Disable all logging.
+
+EXAMPLES
+  Transform the release notes from version 2.2.0 and output the results to out.md.
+
+    $ flub transform releaseNotes --inFile RELEASE_NOTES/2.2.0.md --outFile out.md
+```
+
+_See code: [src/commands/transform/releaseNotes.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/transform/releaseNotes.ts)_

build-tools/packages/build-cli/package.json

@@ -112,6 +112,7 @@
 		"jssm": "5.98.2",
 		"latest-version": "^5.1.0",
 		"mdast": "^3.0.0",
+		"mdast-util-heading-range": "^4.0.0",
 		"mdast-util-to-string": "^4.0.0",
 		"minimatch": "^7.4.6",
 		"node-fetch": "^3.3.2",
@@ -239,6 +240,9 @@
 			},
 			"promote": {
 				"description": "Promote commands are used to promote packages published to an npm registry."
+			},
+			"transform": {
+				"description": "Transform commands are used to transform code, docs, etc. into alternative forms."
 			}
 		}
 	}

build-tools/packages/build-cli/src/commands/generate/releaseNotes.ts

@@ -26,7 +26,9 @@ import {
 	loadChangesets,
 } from "../../library/index.js";
 // eslint-disable-next-line import/no-internal-modules
-import { remarkHeadingLinks, stripSoftBreaks } from "../../library/markdown.js";
+import { addHeadingLinks, stripSoftBreaks } from "../../library/markdown.js";
+// eslint-disable-next-line import/no-internal-modules
+import { RELEASE_NOTES_TOC_LINK_TEXT } from "../../library/releaseNotes.js";
 
 /**
  * Generates release notes from individual changeset files.
@@ -196,7 +198,7 @@ export default class GenerateReleaseNotesCommand extends BaseCommand<
 						.join("");
 					body.append(`Affected packages:\n\n${affectedPackages}\n\n`);
 					body.append(
-						`[⬆️ Table of contents](#${flags.headingLinks ? "user-content-" : ""}contents)\n\n`,
+						`[${RELEASE_NOTES_TOC_LINK_TEXT}](#${flags.headingLinks ? "user-content-" : ""}contents)\n\n`,
 					);
 				} else {
 					this.info(
@@ -233,9 +235,7 @@ export default class GenerateReleaseNotesCommand extends BaseCommand<
 				},
 			});
 
-		const processor = flags.headingLinks
-			? baseProcessor.use(remarkHeadingLinks)
-			: baseProcessor;
+		const processor = flags.headingLinks ? baseProcessor.use(addHeadingLinks) : baseProcessor;
 
 		const contents = String(
 			await processor.process(`${header}\n\n${intro}\n\n${body.toString()}\n\n${footer}`),

build-tools/packages/build-cli/src/commands/transform/releaseNotes.ts

@@ -0,0 +1,111 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { readFile, writeFile } from "node:fs/promises";
+import { Flags } from "@oclif/core";
+import { format as prettier } from "prettier";
+import { remark } from "remark";
+import remarkGfm from "remark-gfm";
+import remarkGithub, { defaultBuildUrl } from "remark-github";
+import admonitions from "remark-github-beta-blockquote-admonitions";
+import remarkToc from "remark-toc";
+
+import { BaseCommand } from "../../library/index.js";
+import {
+	addHeadingLinks,
+	removeHeadingsAtLevel,
+	removeSectionContent,
+	stripSoftBreaks,
+	updateTocLinks,
+	// eslint-disable-next-line import/no-internal-modules
+} from "../../library/markdown.js";
+// eslint-disable-next-line import/no-internal-modules
+import { RELEASE_NOTES_TOC_LINK_TEXT } from "../../library/releaseNotes.js";
+
+/**
+ * Transforms a markdown release notes file into a format appropriate for use in a GitHub Release.
+ */
+export default class TransformReleaseNotesCommand extends BaseCommand<
+	typeof TransformReleaseNotesCommand
+> {
+	static readonly summary =
+		`Transforms a markdown release notes file into a format appropriate for use in a GitHub Release. This is used to transform in-repo release notes such that they can be automatically posted to our GitHub Releases.`;
+
+	static readonly flags = {
+		inFile: Flags.file({
+			description: `A release notes file that was generated using 'flub generate releaseNotes'.`,
+			required: true,
+			exists: true,
+		}),
+		outFile: Flags.file({
+			description: `Output the transformed content to this file.`,
+			required: true,
+		}),
+		...BaseCommand.flags,
+	} as const;
+
+	static readonly examples = [
+		{
+			description: `Transform the release notes from version 2.2.0 and output the results to out.md.`,
+			command:
+				"<%= config.bin %> <%= command.id %> --inFile RELEASE_NOTES/2.2.0.md --outFile out.md",
+		},
+	];
+
+	public async run(): Promise<string> {
+		const { inFile, outFile } = this.flags;
+		const input = await readFile(inFile, { encoding: "utf8" });
+		const processor = remark()
+			// Remove the H1 if it exists.
+			.use(removeHeadingsAtLevel, { level: 1 })
+			// Remove the existing TOC section because its links are incorrect; we'll regenerate it.
+			.use(removeSectionContent, { heading: "Contents" })
+			// Update the "back to TOC" links to prepend 'user-content-' because that's what GH Releases does.
+			.use(updateTocLinks, {
+				checkValue: RELEASE_NOTES_TOC_LINK_TEXT,
+				newUrl: "#user-content-contents",
+			})
+			// Parse the markdown as GitHub-Flavored Markdown
+			.use(remarkGfm)
+			// Strip any single-line breaks. See the docs for the stripSoftBreaks function for more details.
+			.use(stripSoftBreaks)
+			// Parse any GitHub admonitions/alerts/callouts
+			.use(admonitions, {
+				titleTextMap: (title) => ({
+					// By default the `[!` prefix and `]` suffix are removed; we don't want that, so we override the default and
+					// return the title as-is.
+					displayTitle: title,
+					checkedTitle: title,
+				}),
+			})
+			// Regenerate the TOC with the user-content- prefix.
+			.use(remarkToc, {
+				maxDepth: 3,
+				skip: ".*Start Building Today.*",
+				// Add the user-content- prefix to the links when we generate our own headingLinks, because GitHub will
+				// prepend that to all our custom anchor IDs.
+				prefix: "user-content-",
+			})
+			// Transform any issue and commit references into links.
+			.use(remarkGithub, {
+				buildUrl(values) {
+					// Disable linking mentions
+					return values.type === "mention" ? false : defaultBuildUrl(values);
+				},
+			})
+			// Add custom anchor tags with IDs to all the headings.
+			.use(addHeadingLinks);
+
+		const contents = String(await processor.process(input));
+
+		this.info(`Writing output file: ${outFile}`);
+		await writeFile(
+			outFile,
+			await prettier(contents, { proseWrap: "never", parser: "markdown" }),
+		);
+
+		return contents;
+	}
+}

build-tools/packages/build-cli/src/library/markdown.ts

@@ -4,10 +4,11 @@
  */
 
 import GithubSlugger from "github-slugger";
-import type { Heading, Html } from "mdast";
+import type { Heading, Html, Link, Root } from "mdast";
+import { headingRange } from "mdast-util-heading-range";
 import { toString } from "mdast-util-to-string";
-import type { Node } from "unist";
-import { visit } from "unist-util-visit";
+import type { Node, Parent } from "unist";
+import { SKIP, visit } from "unist-util-visit";
 
 /**
  * Using the same instance for all slug generation ensures that no duplicate IDs are generated.
@@ -21,10 +22,16 @@ const slugger = new GithubSlugger();
  *
  * For more details, see: https://github.com/orgs/community/discussions/48311#discussioncomment-10436184
  */
-export function remarkHeadingLinks(): (tree: Node) => void {
+export function addHeadingLinks(): (tree: Node) => void {
 	return (tree: Node): void => {
 		visit(tree, "heading", (node: Heading) => {
-			if (node.children?.length > 0) {
+			if (
+				node.children?.length > 0 &&
+				// This check ensures that we don't add links to headings that already have them. In such cases the first child
+				// node's type will be html, not text. Note that this check could ignore some node types other than text that
+				// would be fine to add headings to, but we've not come across any such cases.
+				node.children[0].type === "text"
+			) {
 				// Calling toString on the whole node ensures that embedded nodes (e.g. formatted text in the heading) are
 				// included in the slugged string.
 				const slug = slugger.slug(toString(node));
@@ -45,7 +52,7 @@ export function remarkHeadingLinks(): (tree: Node) => void {
  * A regular expression that extracts an admonition title from a string UNLESS the admonition title is the only thing on
  * the line.
  *
- * Capture group 1 is the admonition type/title (from the leading `[!` all the way to the trailing `]`). Capture group 2 is any trailing whitespace.
+ * Capture group 1 is the admonition type/title (from the leading `[!` all the way to the trailing `]`).
  *
  * @remarks
  *
@@ -55,7 +62,7 @@ export function remarkHeadingLinks(): (tree: Node) => void {
  * WARNING. It ensures that the pattern is not followed by only whitespace characters until the end of the line.
  * Additionally, it captures any whitespace characters that follow the matched pattern.
  */
-const ADMONITION_REGEX = /(\[!(?:CAUTION|IMPORTANT|NOTE|TIP|WARNING)])(?!\s*$)(\s*)/gm;
+const ADMONITION_REGEX = /(\[!(?:CAUTION|IMPORTANT|NOTE|TIP|WARNING)])(?!\s*$)\s*/gm;
 
 /**
  * A regular expression to remove single line breaks from text. This is used to remove extraneous line breaks in text
@@ -93,3 +100,66 @@ export function stripSoftBreaks(): (tree: Node) => void {
 		});
 	};
 }
+
+/**
+ * Given a heading string or regex, removes all the content in sections under that heading. Most useful for removing a
+ * table of contents section that will later be regenerated. Note that the section heading remains - only the inner
+ * content is removed.
+ *
+ * @param options - `heading` is a string or regex that a section's heading must match to be removed.
+ */
+export function removeSectionContent(options: { heading: string | RegExp }): (
+	tree: Root,
+) => void {
+	return function (tree: Root) {
+		headingRange(tree, options.heading, (start, nodes, end, info) => {
+			return [
+				start,
+				// No child nodes - effectively empties the section.
+				end,
+			];
+		});
+	};
+}
+
+/**
+ * Removes all the headings at a particular level. Most useful to remove the top-level H1 headings from a document.
+ *
+ * @param options - The `level` property must be set to the level of heading to remove.
+ */
+export function removeHeadingsAtLevel(options: { level: 1 | 2 | 3 | 4 | 5 | 6 }): (
+	tree: Root,
+) => void {
+	return (tree: Root) => {
+		visit(
+			tree,
+			"heading",
+			(node: Heading, index: number | undefined, parent: Parent | undefined) => {
+				if (node.depth === options.level && index !== undefined) {
+					parent?.children.splice(index, 1);
+					return [SKIP, index];
+				}
+			},
+		);
+	};
+}
+
+/**
+ * Updates URLs of links whose value match a provided value.
+ *
+ * @param options - `checkValue` is a string that will be compared against the link text. Only matching nodes will be
+ * updated. `newUrl` is the new URL to assign to the link.
+ */
+export function updateTocLinks(options: { checkValue: string; newUrl: string }): (
+	tree: Root,
+) => void {
+	const { checkValue, newUrl } = options;
+
+	return (tree: Root) => {
+		visit(tree, "link", (node: Link) => {
+			if (node.children?.[0].type === "text" && node.children[0].value === checkValue) {
+				node.url = newUrl;
+			}
+		});
+	};
+}

build-tools/packages/build-cli/src/library/releaseNotes.ts

@@ -0,0 +1,9 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/**
+ * The text used in release notes links that point back to the table of contents in the document.
+ */
+export const RELEASE_NOTES_TOC_LINK_TEXT = "⬆️ Table of contents";