feat(build-cli): New command transform:releaseNotes

build-tools/packages/build-cli/README.md

@@ -48,6 +48,7 @@ USAGE
 * [`flub release`](docs/release.md) - Release commands are used to manage the Fluid release process.
 * [`flub rename-types`](docs/rename-types.md) - Renames type declaration files from .d.ts to .d.mts.
 * [`flub run`](docs/run.md) - Generate a report from input bundle stats collected through the collect bundleStats command.
+* [`flub transform`](docs/transform.md) - Transform commands are used to transform code, docs, etc. into alternative forms.
 * [`flub typetests`](docs/typetests.md) - Updates configuration for type tests in package.json files. If the previous version changes after running preparation, then npm install must be run before building.
 
 <!-- commandsstop -->

build-tools/packages/build-cli/docs/transform.md

@@ -0,0 +1,30 @@
+`flub transform`
+================
+
+Transform commands are used to transform code, docs, etc. into alternative forms.
+
+* [`flub transform releaseNotes`](#flub-transform-releasenotes)
+
+## `flub transform releaseNotes`
+
+Transforms a markdown release notes file into a format appropriate for use in a GitHub Release. This is used to transform in-repo release notes such that they can be automatically posted to our GitHub Releases.
+
+```
+USAGE
+  $ flub transform releaseNotes --inFile <value> --outFile <value> [-v | --quiet]
+
+FLAGS
+  --inFile=<value>   (required) A release notes file that was generated using 'flub generate releaseNotes'.
+  --outFile=<value>  (required) Output the transformed content to this file.
+
+LOGGING FLAGS
+  -v, --verbose  Enable verbose logging.
+      --quiet    Disable all logging.
+
+EXAMPLES
+  Transform the release notes from version 2.2.0 and output the results to out.md.
+
+    $ flub transform releaseNotes --inFile RELEASE_NOTES/2.2.0.md --outFile out.md
+```
+
+_See code: [src/commands/transform/releaseNotes.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/transform/releaseNotes.ts)_

build-tools/packages/build-cli/package.json

@@ -112,6 +112,7 @@
 		"jssm": "5.98.2",
 		"latest-version": "^5.1.0",
 		"mdast": "^3.0.0",
+		"mdast-util-heading-range": "^4.0.0",
 		"mdast-util-to-string": "^4.0.0",
 		"minimatch": "^7.4.6",
 		"node-fetch": "^3.3.2",
@@ -239,6 +240,9 @@
 			},
 			"promote": {
 				"description": "Promote commands are used to promote packages published to an npm registry."
+			},
+			"transform": {
+				"description": "Transform commands are used to transform code, docs, etc. into alternative forms."
 			}
 		}
 	}

build-tools/packages/build-cli/src/commands/generate/releaseNotes.ts

@@ -26,7 +26,9 @@ import {
 	loadChangesets,
 } from "../../library/index.js";
 // eslint-disable-next-line import/no-internal-modules
-import { remarkHeadingLinks, stripSoftBreaks } from "../../library/markdown.js";
+import { addHeadingLinks, stripSoftBreaks } from "../../library/markdown.js";
+// eslint-disable-next-line import/no-internal-modules
+import { RELEASE_NOTES_TOC_LINK_TEXT } from "../../library/releaseNotes.js";
 
 /**
  * Generates release notes from individual changeset files.
@@ -196,7 +198,7 @@ export default class GenerateReleaseNotesCommand extends BaseCommand<
 						.join("");
 					body.append(`Affected packages:\n\n${affectedPackages}\n\n`);
 					body.append(
-						`[⬆️ Table of contents](#${flags.headingLinks ? "user-content-" : ""}contents)\n\n`,
+						`[${RELEASE_NOTES_TOC_LINK_TEXT}](#${flags.headingLinks ? "user-content-" : ""}contents)\n\n`,
 					);
 				} else {
 					this.info(
@@ -233,9 +235,7 @@ export default class GenerateReleaseNotesCommand extends BaseCommand<
 				},
 			});
 
-		const processor = flags.headingLinks
-			? baseProcessor.use(remarkHeadingLinks)
-			: baseProcessor;
+		const processor = flags.headingLinks ? baseProcessor.use(addHeadingLinks) : baseProcessor;
 
 		const contents = String(
 			await processor.process(`${header}\n\n${intro}\n\n${body.toString()}\n\n${footer}`),

build-tools/packages/build-cli/src/commands/transform/releaseNotes.ts

@@ -0,0 +1,111 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { readFile, writeFile } from "node:fs/promises";
+import { Flags } from "@oclif/core";
+import { format as prettier } from "prettier";
+import { remark } from "remark";
+import remarkGfm from "remark-gfm";
+import remarkGithub, { defaultBuildUrl } from "remark-github";
+import admonitions from "remark-github-beta-blockquote-admonitions";
+import remarkToc from "remark-toc";
+
+import { BaseCommand } from "../../library/index.js";
+import {
+	addHeadingLinks,
+	removeHeadingsAtLevel,
+	removeSectionContent,
+	stripSoftBreaks,
+	updateTocLinks,
+	// eslint-disable-next-line import/no-internal-modules
+} from "../../library/markdown.js";
+// eslint-disable-next-line import/no-internal-modules
+import { RELEASE_NOTES_TOC_LINK_TEXT } from "../../library/releaseNotes.js";
+
+/**
+ * Transforms a markdown release notes file into a format appropriate for use in a GitHub Release.
+ */
+export default class TransformReleaseNotesCommand extends BaseCommand<
+	typeof TransformReleaseNotesCommand
+> {
+	static readonly summary =
+		`Transforms a markdown release notes file into a format appropriate for use in a GitHub Release. This is used to transform in-repo release notes such that they can be automatically posted to our GitHub Releases.`;
+
+	static readonly flags = {
+		inFile: Flags.file({
+			description: `A release notes file that was generated using 'flub generate releaseNotes'.`,
+			required: true,
+			exists: true,
+		}),
+		outFile: Flags.file({
+			description: `Output the transformed content to this file.`,
+			required: true,
+		}),
+		...BaseCommand.flags,
+	} as const;
+
+	static readonly examples = [
+		{
+			description: `Transform the release notes from version 2.2.0 and output the results to out.md.`,
+			command:
+				"<%= config.bin %> <%= command.id %> --inFile RELEASE_NOTES/2.2.0.md --outFile out.md",
+		},
+	];
+
+	public async run(): Promise<string> {
+		const { inFile, outFile } = this.flags;
+		const input = await readFile(inFile, { encoding: "utf8" });
+		const processor = remark()
+			// Remove the H1 if it exists.
+			.use(removeHeadingsAtLevel, { level: 1 })
+			// Remove the existing TOC section because its links are incorrect; we'll regenerate it.
+			.use(removeSectionContent, { heading: "Contents" })
+			// Update the "back to TOC" links to prepend 'user-content-' because that's what GH Releases does.
+			.use(updateTocLinks, {
+				checkValue: RELEASE_NOTES_TOC_LINK_TEXT,
+				newUrl: "#user-content-contents",
+			})
+			// Parse the markdown as GitHub-Flavored Markdown
+			.use(remarkGfm)
+			// Strip any single-line breaks. See the docs for the stripSoftBreaks function for more details.
+			.use(stripSoftBreaks)
+			// Parse any GitHub admonitions/alerts/callouts
+			.use(admonitions, {
+				titleTextMap: (title) => ({
+					// By default the `[!` prefix and `]` suffix are removed; we don't want that, so we override the default and
+					// return the title as-is.
+					displayTitle: title,
+					checkedTitle: title,
+				}),
+			})
+			// Regenerate the TOC with the user-content- prefix.
+			.use(remarkToc, {
+				maxDepth: 3,
+				skip: ".*Start Building Today.*",
+				// Add the user-content- prefix to the links when we generate our own headingLinks, because GitHub will
+				// prepend that to all our custom anchor IDs.
+				prefix: "user-content-",
+			})
+			// Transform any issue and commit references into links.
+			.use(remarkGithub, {
+				buildUrl(values) {
+					// Disable linking mentions
+					return values.type === "mention" ? false : defaultBuildUrl(values);
+				},
+			})
+			// Add custom anchor tags with IDs to all the headings.
+			.use(addHeadingLinks);
+
+		const contents = String(await processor.process(input));
+
+		this.info(`Writing output file: ${outFile}`);
+		await writeFile(
+			outFile,
+			await prettier(contents, { proseWrap: "never", parser: "markdown" }),
+		);
+
+		return contents;
+	}
+}

build-tools/packages/build-cli/src/library/markdown.ts

@@ -4,10 +4,11 @@
  */
 
 import GithubSlugger from "github-slugger";
-import type { Heading, Html } from "mdast";
+import type { Heading, Html, Link, Root } from "mdast";
+import { headingRange } from "mdast-util-heading-range";
 import { toString } from "mdast-util-to-string";
-import type { Node } from "unist";
-import { visit } from "unist-util-visit";
+import type { Node, Parent } from "unist";
+import { SKIP, visit } from "unist-util-visit";
 
 /**
  * Using the same instance for all slug generation ensures that no duplicate IDs are generated.
@@ -21,10 +22,16 @@ const slugger = new GithubSlugger();
  *
  * For more details, see: https://github.com/orgs/community/discussions/48311#discussioncomment-10436184
  */
-export function remarkHeadingLinks(): (tree: Node) => void {
+export function addHeadingLinks(): (tree: Node) => void {
 	return (tree: Node): void => {
 		visit(tree, "heading", (node: Heading) => {
-			if (node.children?.length > 0) {
+			if (
+				node.children?.length > 0 &&
+				// This check ensures that we don't add links to headings that already have them. In such cases the first child
+				// node's type will be html, not text. Note that this check could ignore some node types other than text that
+				// would be fine to add headings to, but we've not come across any such cases.
+				node.children[0].type === "text"
+			) {
 				// Calling toString on the whole node ensures that embedded nodes (e.g. formatted text in the heading) are
 				// included in the slugged string.
 				const slug = slugger.slug(toString(node));
@@ -45,7 +52,7 @@ export function remarkHeadingLinks(): (tree: Node) => void {
  * A regular expression that extracts an admonition title from a string UNLESS the admonition title is the only thing on
  * the line.
  *
- * Capture group 1 is the admonition type/title (from the leading `[!` all the way to the trailing `]`). Capture group 2 is any trailing whitespace.
+ * Capture group 1 is the admonition type/title (from the leading `[!` all the way to the trailing `]`).
  *
  * @remarks
  *
@@ -55,7 +62,7 @@ export function remarkHeadingLinks(): (tree: Node) => void {
  * WARNING. It ensures that the pattern is not followed by only whitespace characters until the end of the line.
  * Additionally, it captures any whitespace characters that follow the matched pattern.
  */
-const ADMONITION_REGEX = /(\[!(?:CAUTION|IMPORTANT|NOTE|TIP|WARNING)])(?!\s*$)(\s*)/gm;
+const ADMONITION_REGEX = /(\[!(?:CAUTION|IMPORTANT|NOTE|TIP|WARNING)])(?!\s*$)\s*/gm;
 
 /**
  * A regular expression to remove single line breaks from text. This is used to remove extraneous line breaks in text
@@ -93,3 +100,66 @@ export function stripSoftBreaks(): (tree: Node) => void {
 		});
 	};
 }
+
+/**
+ * Given a heading string or regex, removes all the content in sections under that heading. Most useful for removing a
+ * table of contents section that will later be regenerated. Note that the section heading remains - only the inner
+ * content is removed.
+ *
+ * @param options - `heading` is a string or regex that a section's heading must match to be removed.
+ */
+export function removeSectionContent(options: { heading: string | RegExp }): (
+	tree: Root,
+) => void {
+	return function (tree: Root) {
+		headingRange(tree, options.heading, (start, nodes, end, info) => {
+			return [
+				start,
+				// No child nodes - effectively empties the section.
+				end,
+			];
+		});
+	};
+}
+
+/**
+ * Removes all the headings at a particular level. Most useful to remove the top-level H1 headings from a document.
+ *
+ * @param options - The `level` property must be set to the level of heading to remove.
+ */
+export function removeHeadingsAtLevel(options: { level: 1 | 2 | 3 | 4 | 5 | 6 }): (
+	tree: Root,
+) => void {
+	return (tree: Root) => {
+		visit(
+			tree,
+			"heading",
+			(node: Heading, index: number | undefined, parent: Parent | undefined) => {
+				if (node.depth === options.level && index !== undefined) {
+					parent?.children.splice(index, 1);
+					return [SKIP, index];
+				}
+			},
+		);
+	};
+}
+
+/**
+ * Updates URLs of links whose value match a provided value.
+ *
+ * @param options - `checkValue` is a string that will be compared against the link text. Only matching nodes will be
+ * updated. `newUrl` is the new URL to assign to the link.
+ */
+export function updateTocLinks(options: { checkValue: string; newUrl: string }): (
+	tree: Root,
+) => void {
+	const { checkValue, newUrl } = options;
+
+	return (tree: Root) => {
+		visit(tree, "link", (node: Link) => {
+			if (node.children?.[0].type === "text" && node.children[0].value === checkValue) {
+				node.url = newUrl;
+			}
+		});
+	};
+}

build-tools/packages/build-cli/src/library/releaseNotes.ts

@@ -0,0 +1,9 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/**
+ * The text used in release notes links that point back to the table of contents in the document.
+ */
+export const RELEASE_NOTES_TOC_LINK_TEXT = "⬆️ Table of contents";