feat(build-cli): New command transform:releaseNotes

.github/workflows/data/release-notes-issue-intro.njk

@@ -12,6 +12,6 @@ pnpm flub generate releaseNotes -g client -t minor --outFile RELEASE_NOTES/{{ ve
 pnpm flub generate releaseNotes -g client -t minor --headingLinks --excludeH1 --outFile temporary-file.md
 ```
 
-You can then copy and paste the contents of the `temporary-file.md` into the GitHub Release.
+This should be done automatically as part of the release process, but if you need to generate the release notes manually, you can use the above command.
 
 ---

.github/workflows/push-tag-create-release.yml

@@ -72,7 +72,7 @@ jobs:
         with:
           name: release-metadata
           path: release-metadata.json
-          retention-days: 3
+          retention-days: 30
       - name: Load release metadata into env variable
         run: |
           echo "RELEASE_JSON=$(cat release-metadata.json)" >> $GITHUB_ENV
@@ -101,10 +101,10 @@ jobs:
           path: reports
           retention-days: 7
 
-      # Generate changelog
-      - name: Generate changelog
-        # This changelog is only for client patch releases and build-tools releases
-        if: (fromJson(env.RELEASE_JSON).releaseType == 'patch' && fromJson(env.RELEASE_JSON).packageOrReleaseGroup == 'client') || fromJson(env.RELEASE_JSON).packageOrReleaseGroup == 'build-tools'
+      # Generate release notes
+      - name: Generate patch release notes
+        # This content is only for patch releases
+        if: fromJson(env.RELEASE_JSON).releaseType == 'patch'
         run: |
           # We only need the root dependencies
           pnpm install -w --frozen-lockfile
@@ -116,11 +116,16 @@ jobs:
           --tag-prefix ${{ fromJson(env.RELEASE_JSON).packageOrReleaseGroup }}_v \
           --output auto-changelog.md \
           --template .github/workflows/data/patch-changelog.hbs
-      - name: Generate changelog
-        # This changelog is a basically empty one used for everything except client patches and build-tools.
-        if: ${{ !(fromJson(env.RELEASE_JSON).releaseType == 'patch' && fromJson(env.RELEASE_JSON).packageOrReleaseGroup == 'client') && fromJson(env.RELEASE_JSON).packageOrReleaseGroup != 'build-tools' }}
+
+      - name: Generate minor/major release notes
+        # This content is the "default" release notes - this should be used for minor and major releases of client and
+        # server release groups. To use this, a release group needs to be using changesets. Build-tools does not use
+        # changesets, so it is excluded.
+        if: fromJson(env.RELEASE_JSON).releaseType != 'patch' && (fromJson(env.RELEASE_JSON).packageOrReleaseGroup == 'client' || fromJson(env.RELEASE_JSON).packageOrReleaseGroup == 'server')
         run: |
-          echo "This is a **${{ fromJson(env.RELEASE_JSON).releaseType }}** release." > auto-changelog.md
+          flub publish releaseNotes \
+          --inFile RELEASE_NOTES/${{ fromJson(env.RELEASE_JSON).version }}.md
+          --outFile auto-changelog.md
 
       # Only creates GH releases for client, server, and build-tools releases.
       - name: Create GH release
@@ -133,11 +138,6 @@ jobs:
           # Will skip if a published (non-draft) release already exists.
           skipIfReleaseExists: true
 
-          # If the release is NOT a patch, leave it as a draft. Only patch releases are auto-published because their
-          # auto-generated release notes are sufficient for those releases. Minor and major releases currently require
-          # some curation of the release notes.
-          draft: ${{ fromJson(env.RELEASE_JSON).releaseType != 'patch' }}
-
           # Don't change the draft state when updating an existing release. This setting is not really necessary for us
           # in most cases because we don't pre-create releases, so this workflow always creates a new GH release. It's
           # included mostly for safety reasons, to ensure that existing drafts aren't published accidentally.

build-tools/packages/build-cli/README.md

@@ -48,6 +48,7 @@ USAGE
 * [`flub release`](docs/release.md) - Release commands are used to manage the Fluid release process.
 * [`flub rename-types`](docs/rename-types.md) - Renames type declaration files from .d.ts to .d.mts.
 * [`flub run`](docs/run.md) - Generate a report from input bundle stats collected through the collect bundleStats command.
+* [`flub transform`](docs/transform.md) - Transform commands are used to transform code, docs, etc. into alternative forms.
 * [`flub typetests`](docs/typetests.md) - Updates configuration for type tests in package.json files. If the previous version changes after running preparation, then npm install must be run before building.
 
 <!-- commandsstop -->

build-tools/packages/build-cli/docs/transform.md

@@ -0,0 +1,30 @@
+`flub transform`
+================
+
+Transform commands are used to transform code, docs, etc. into alternative forms.
+
+* [`flub transform releaseNotes`](#flub-transform-releasenotes)
+
+## `flub transform releaseNotes`
+
+Transforms a markdown release notes file into a format appropriate for use in a GitHub Release. This is used to transform in-repo release notes such that they can be automatically posted to our GitHub Releases.
+
+```
+USAGE
+  $ flub transform releaseNotes --inFile <value> --outFile <value> [-v | --quiet]
+
+FLAGS
+  --inFile=<value>   (required) A release notes file that was generated using 'flub generate releaseNotes'.
+  --outFile=<value>  (required) Output the transformed content to this file.
+
+LOGGING FLAGS
+  -v, --verbose  Enable verbose logging.
+      --quiet    Disable all logging.
+
+EXAMPLES
+  Transform the release notes from version 2.2.0 and output the results to out.md.
+
+    $ flub transform releaseNotes --inFile RELEASE_NOTES/2.2.0.md --outFile out.md
+```
+
+_See code: [src/commands/transform/releaseNotes.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/transform/releaseNotes.ts)_

build-tools/packages/build-cli/package.json

@@ -112,6 +112,7 @@
 		"jssm": "5.98.2",
 		"latest-version": "^5.1.0",
 		"mdast": "^3.0.0",
+		"mdast-util-heading-range": "^4.0.0",
 		"mdast-util-to-string": "^4.0.0",
 		"minimatch": "^7.4.6",
 		"node-fetch": "^3.3.2",
@@ -136,6 +137,7 @@
 		"table": "^6.8.1",
 		"ts-morph": "^22.0.0",
 		"type-fest": "^2.19.0",
+		"unified": "^11.0.5",
 		"unist-util-visit": "^5.0.0"
 	},
 	"devDependencies": {
@@ -239,6 +241,9 @@
 			},
 			"promote": {
 				"description": "Promote commands are used to promote packages published to an npm registry."
+			},
+			"transform": {
+				"description": "Transform commands are used to transform code, docs, etc. into alternative forms."
 			}
 		}
 	}

build-tools/packages/build-cli/src/commands/generate/releaseNotes.ts

@@ -26,7 +26,9 @@ import {
 	loadChangesets,
 } from "../../library/index.js";
 // eslint-disable-next-line import/no-internal-modules
-import { remarkHeadingLinks, stripSoftBreaks } from "../../library/markdown.js";
+import { addHeadingLinks, stripSoftBreaks } from "../../library/markdown.js";
+// eslint-disable-next-line import/no-internal-modules
+import { RELEASE_NOTES_TOC_LINK_TEXT } from "../../library/releaseNotes.js";
 
 /**
  * Generates release notes from individual changeset files.
@@ -196,7 +198,7 @@ export default class GenerateReleaseNotesCommand extends BaseCommand<
 						.join("");
 					body.append(`Affected packages:\n\n${affectedPackages}\n\n`);
 					body.append(
-						`[⬆️ Table of contents](#${flags.headingLinks ? "user-content-" : ""}contents)\n\n`,
+						`[${RELEASE_NOTES_TOC_LINK_TEXT}](#${flags.headingLinks ? "user-content-" : ""}contents)\n\n`,
 					);
 				} else {
 					this.info(
@@ -233,9 +235,7 @@ export default class GenerateReleaseNotesCommand extends BaseCommand<
 				},
 			});
 
-		const processor = flags.headingLinks
-			? baseProcessor.use(remarkHeadingLinks)
-			: baseProcessor;
+		const processor = flags.headingLinks ? baseProcessor.use(addHeadingLinks) : baseProcessor;
 
 		const contents = String(
 			await processor.process(`${header}\n\n${intro}\n\n${body.toString()}\n\n${footer}`),

build-tools/packages/build-cli/src/commands/transform/releaseNotes.ts

@@ -0,0 +1,111 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { readFile, writeFile } from "node:fs/promises";
+import { Flags } from "@oclif/core";
+import { format as prettier } from "prettier";
+import { remark } from "remark";
+import remarkGfm from "remark-gfm";
+import remarkGithub, { defaultBuildUrl } from "remark-github";
+import admonitions from "remark-github-beta-blockquote-admonitions";
+import remarkToc from "remark-toc";
+
+import { BaseCommand } from "../../library/index.js";
+import {
+	addHeadingLinks,
+	removeHeadingsAtLevel,
+	removeSectionContent,
+	stripSoftBreaks,
+	updateTocLinks,
+	// eslint-disable-next-line import/no-internal-modules
+} from "../../library/markdown.js";
+// eslint-disable-next-line import/no-internal-modules
+import { RELEASE_NOTES_TOC_LINK_TEXT } from "../../library/releaseNotes.js";
+
+/**
+ * Transforms a markdown release notes file into a format appropriate for use in a GitHub Release.
+ */
+export default class TransformReleaseNotesCommand extends BaseCommand<
+	typeof TransformReleaseNotesCommand
+> {
+	static readonly summary =
+		`Transforms a markdown release notes file into a format appropriate for use in a GitHub Release. This is used to transform in-repo release notes such that they can be automatically posted to our GitHub Releases.`;
+
+	static readonly flags = {
+		inFile: Flags.file({
+			description: `A release notes file that was generated using 'flub generate releaseNotes'.`,
+			required: true,
+			exists: true,
+		}),
+		outFile: Flags.file({
+			description: `Output the transformed content to this file.`,
+			required: true,
+		}),
+		...BaseCommand.flags,
+	} as const;
+
+	static readonly examples = [
+		{
+			description: `Transform the release notes from version 2.2.0 and output the results to out.md.`,
+			command:
+				"<%= config.bin %> <%= command.id %> --inFile RELEASE_NOTES/2.2.0.md --outFile out.md",
+		},
+	];
+
+	public async run(): Promise<string> {
+		const { inFile, outFile } = this.flags;
+		const input = await readFile(inFile, { encoding: "utf8" });
+		const processor = remark()
+			// Remove the H1 if it exists.
+			.use(removeHeadingsAtLevel, { level: 1 })
+			// Remove the existing TOC section because its links are incorrect; we'll regenerate it.
+			.use(removeSectionContent, { heading: "Contents" })
+			// Update the "back to TOC" links to prepend 'user-content-' because that's what GH Releases does.
+			.use(updateTocLinks, {
+				checkValue: RELEASE_NOTES_TOC_LINK_TEXT,
+				newUrl: "#user-content-contents",
+			})
+			// Parse the markdown as GitHub-Flavored Markdown
+			.use(remarkGfm)
+			// Strip any single-line breaks. See the docs for the stripSoftBreaks function for more details.
+			.use(stripSoftBreaks)
+			// Parse any GitHub admonitions/alerts/callouts
+			.use(admonitions, {
+				titleTextMap: (title) => ({
+					// By default the `[!` prefix and `]` suffix are removed; we don't want that, so we override the default and
+					// return the title as-is.
+					displayTitle: title,
+					checkedTitle: title,
+				}),
+			})
+			// Regenerate the TOC with the user-content- prefix.
+			.use(remarkToc, {
+				maxDepth: 3,
+				skip: ".*Start Building Today.*",
+				// Add the user-content- prefix to the links when we generate our own headingLinks, because GitHub will
+				// prepend that to all our custom anchor IDs.
+				prefix: "user-content-",
+			})
+			// Transform any issue and commit references into links.
+			.use(remarkGithub, {
+				buildUrl(values) {
+					// Disable linking mentions
+					return values.type === "mention" ? false : defaultBuildUrl(values);
+				},
+			})
+			// Add custom anchor tags with IDs to all the headings.
+			.use(addHeadingLinks);
+
+		const contents = String(await processor.process(input));
+
+		this.info(`Writing output file: ${outFile}`);
+		await writeFile(
+			outFile,
+			await prettier(contents, { proseWrap: "never", parser: "markdown" }),
+		);
+
+		return contents;
+	}
+}

build-tools/packages/build-cli/src/library/markdown.ts

@@ -4,10 +4,11 @@
  */
 
 import GithubSlugger from "github-slugger";
-import type { Heading, Html } from "mdast";
+import type { Heading, Html, Link, Root } from "mdast";
+import { headingRange } from "mdast-util-heading-range";
 import { toString } from "mdast-util-to-string";
-import type { Node } from "unist";
-import { visit } from "unist-util-visit";
+import type { Node, Parent } from "unist";
+import { SKIP, visit } from "unist-util-visit";
 
 /**
  * Using the same instance for all slug generation ensures that no duplicate IDs are generated.
@@ -21,10 +22,10 @@ const slugger = new GithubSlugger();
  *
  * For more details, see: https://github.com/orgs/community/discussions/48311#discussioncomment-10436184
  */
-export function remarkHeadingLinks(): (tree: Node) => void {
+export function addHeadingLinks(): (tree: Node) => void {
 	return (tree: Node): void => {
 		visit(tree, "heading", (node: Heading) => {
-			if (node.children?.length > 0) {
+			if (node.children?.length > 0 && node.children[0].type === "text") {
 				// Calling toString on the whole node ensures that embedded nodes (e.g. formatted text in the heading) are
 				// included in the slugged string.
 				const slug = slugger.slug(toString(node));
@@ -45,7 +46,7 @@ export function remarkHeadingLinks(): (tree: Node) => void {
  * A regular expression that extracts an admonition title from a string UNLESS the admonition title is the only thing on
  * the line.
  *
- * Capture group 1 is the admonition type/title (from the leading `[!` all the way to the trailing `]`). Capture group 2 is any trailing whitespace.
+ * Capture group 1 is the admonition type/title (from the leading `[!` all the way to the trailing `]`).
  *
  * @remarks
  *
@@ -55,7 +56,7 @@ export function remarkHeadingLinks(): (tree: Node) => void {
  * WARNING. It ensures that the pattern is not followed by only whitespace characters until the end of the line.
  * Additionally, it captures any whitespace characters that follow the matched pattern.
  */
-const ADMONITION_REGEX = /(\[!(?:CAUTION|IMPORTANT|NOTE|TIP|WARNING)])(?!\s*$)(\s*)/gm;
+const ADMONITION_REGEX = /(\[!(?:CAUTION|IMPORTANT|NOTE|TIP|WARNING)])(?!\s*$)\s*/gm;
 
 /**
  * A regular expression to remove single line breaks from text. This is used to remove extraneous line breaks in text
@@ -93,3 +94,66 @@ export function stripSoftBreaks(): (tree: Node) => void {
 		});
 	};
 }
+
+/**
+ * Given a heading string or regex, removes all the content in sections under that heading. Most useful for removing a
+ * table of contents section that will later be regenerated. Note that the section heading remains - only the inner
+ * content is removed.
+ *
+ * @param options - `heading` is a string or regex that a section's heading must match to be removed.
+ */
+export function removeSectionContent(options: { heading: string | RegExp }): (
+	tree: Root,
+) => void {
+	return function (tree: Root) {
+		headingRange(tree, options.heading, (start, nodes, end, info) => {
+			return [
+				start,
+				// No child nodes - effectively empties the section.
+				end,
+			];
+		});
+	};
+}
+
+/**
+ * Removes all the headings at a particular level. Most useful to remove the top-level H1 headings from a document.
+ *
+ * @param options - The `level` property must be set to the level of heading to remove.
+ */
+export function removeHeadingsAtLevel(options: { level: 1 | 2 | 3 | 4 | 5 | 6 }): (
+	tree: Root,
+) => void {
+	return (tree: Root) => {
+		visit(
+			tree,
+			"heading",
+			(node: Heading, index: number | undefined, parent: Parent | undefined) => {
+				if (node.depth === options.level && index !== undefined) {
+					parent?.children.splice(index, 1);
+					return [SKIP, index];
+				}
+			},
+		);
+	};
+}
+
+/**
+ * Updates URLs of links whose value match a provided value.
+ *
+ * @param options - `checkValue` is a string that will be compared against the link text. Only matching nodes will be
+ * updated. `newUrl` is the new URL to assign to the link.
+ */
+export function updateTocLinks(options: { checkValue: string; newUrl: string }): (
+	tree: Root,
+) => void {
+	const { checkValue, newUrl } = options;
+
+	return (tree: Root) => {
+		visit(tree, "link", (node: Link) => {
+			if (node.children?.[0].type === "text" && node.children[0].value === checkValue) {
+				node.url = newUrl;
+			}
+		});
+	};
+}

build-tools/packages/build-cli/src/library/releaseNotes.ts

@@ -0,0 +1,9 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/**
+ * The text used in release notes links that point back to the table of contents in the document.
+ */
+export const RELEASE_NOTES_TOC_LINK_TEXT = "⬆️ Table of contents";