feat(api-markdown-documenter): Add transformation logic for convertin…

…g our `DocumentationDomain` trees to `hast` (#20168)

HTML rendering path does not use this yet, due to issues formatting
inconsistencies using `hast-util-to-html`. I'll investigate these more
in the future. But the core transformation logic works, is tested, and
could be used with other hast-compatible libraries.

### Notes for reviewers

The bulk of the logic (everything under `default-transformations` is
adapted from the code under `html-renderer/default-renderers`. It's
virtually the same code, just with a different output format.

Unfortunately, the "old" codepath still needs to live until some of the
below-mentioned future work can be completed.

### Future work

1. Replace the majority of the HTML render codepath with this new
transformation + `hast-util-to-html` (or something equivalent) - just
need to figure out a solution to the output formatting issues.
2. Create comparable transformation logic to output `mdast` for
Markdown, and update the corresponding renderer to leverage it.
* Required the `toHtml` transformation as a prerequisite, as we leverage
embedded HTML in our Markdown output in certain cases.

tools/api-markdown-documenter/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## 0.12.0
 
+-   Added functionality for transforming Documentation Domain trees to [hast](https://github.com/syntax-tree/hast).
+    The main entrypoint for this is: `documentToHtml`.
 -   Updated the package to emit ESM only.
 -   Fixed a bug where [inline tags](https://tsdoc.org/pages/spec/tag_kinds/#inline-tags) (other than `{@link}` and `{@inheritDoc}`, which are handled specially by API-Extractor) were not handled and resulted in errors being logged to the console.
     Such tags are now handled in the following way:

tools/api-markdown-documenter/README.md

@@ -123,13 +123,21 @@ But this is broken into the following internal sequences:
 graph LR
     A[ApiModel]
     B[Documentation AST]
-    C[Markdown]
+    C[raw Markdown]
+    D[raw HTML]
+    E[HTML AST]
 
-    A -- transformApiModel --> B
-    B -- renderDocumentAsMarkdown --> C
-    A -.- renderApiModelAsMarkdown -.-> C
+    A-->|transformApiModel|B
+    B-->|MarkdownRenderer.renderDocument|C
+    B-->|HtmlRenderer.renderDocument*|D
+    B-->|documentToHtml*|E
+
+    A-.->|MarkdownRenderer.renderApiModel|C
+    A-.->|HtmlRenderer.renderApiModel*|D
 ```
 
+**Note:** APIs above marked with an `*` are in preview, and may change without notice.
+
 For more details on the interior `Documentation AST` ([Abstract Syntax Tree][]) domain, see [Documentation Domain](#documentation-domain) below.
 
 ### API-Extractor
@@ -167,6 +175,11 @@ If you would like to add rendering support for a custom `Documentation Domain` n
 
 If you would like to change any or all of this library's default rendering policies, you may simply override the default policies for the desired `type`s.
 
+## ToHtml Transformation
+
+This library now includes preview APIs for transforming `Documentation Domain` trees to HTML syntax trees using [hast](https://github.com/syntax-tree/hast).
+The main entry-point for this functionality is `documentToHtml`.
+
 ## HtmlRenderer
 
 This library now includes preview APIs for HTML rendering.

tools/api-markdown-documenter/api-report/api-markdown-documenter.api.md

@@ -30,8 +30,10 @@ import { DocSection } from '@microsoft/tsdoc';
 import type { Literal } from 'unist';
 import { NewlineKind } from '@rushstack/node-core-library';
 import type { Node as Node_2 } from 'unist';
+import type { Nodes } from 'hast';
 import type { Parent } from 'unist';
 import { ReleaseTag } from '@microsoft/api-extractor-model';
+import type { Root } from 'hast';
 import { TypeParameter } from '@microsoft/api-extractor-model';
 
 // @public
@@ -213,6 +215,12 @@ export interface DocumentationNode<TData extends object = Data> extends Node_2<T
     readonly type: string;
 }
 
+// @alpha
+export function documentationNodesToHtml(nodes: DocumentationNode[], context: ToHtmlContext): Nodes[];
+
+// @alpha
+export function documentationNodeToHtml(node: DocumentationNode, context: ToHtmlContext): Nodes;
+
 // @public
 export enum DocumentationNodeType {
     BlockQuote = "BlockQuote",
@@ -258,6 +266,7 @@ export abstract class DocumentationParentNodeBase<TDocumentationNode extends Doc
 // @public
 export interface DocumentationSuiteOptions {
     documentBoundaries?: DocumentBoundaries;
+    // @deprecated
     frontMatter?: string | ((documentItem: ApiItem) => string | undefined);
     getFileNameForItem?: (apiItem: ApiItem) => string;
     getHeadingTextForItem?: (apiItem: ApiItem) => string;
@@ -288,9 +297,13 @@ export interface DocumentNodeProps {
     readonly apiItem?: ApiItem;
     readonly children: SectionNode[];
     readonly documentPath: string;
+    // @deprecated
     readonly frontMatter?: string;
 }
 
+// @alpha
+export function documentToHtml(document: DocumentNode, config: ToHtmlConfig): Root;
+
 // @public
 export interface DocumentWriter {
     decreaseIndent(): void;
@@ -550,7 +563,6 @@ export class OrderedListNode extends DocumentationParentNodeBase<SingleLineDocum
 // @public
 export class ParagraphNode extends DocumentationParentNodeBase implements MultiLineDocumentationNode {
     constructor(children: DocumentationNode[]);
-    static combine(...nodes: ParagraphNode[]): ParagraphNode;
     static createFromPlainText(text: string): ParagraphNode;
     static readonly Empty: ParagraphNode;
     get singleLine(): false;
@@ -702,6 +714,27 @@ export interface TextFormatting {
     readonly strikethrough?: boolean;
 }
 
+// @alpha
+export interface ToHtmlConfig extends ConfigurationBase {
+    readonly customTransformations?: ToHtmlTransformations;
+    readonly language?: string;
+    readonly startingHeadingLevel?: number;
+}
+
+// @alpha
+export interface ToHtmlContext extends ConfigurationBase {
+    readonly headingLevel: number;
+    readonly transformations: ToHtmlTransformations;
+}
+
+// @alpha
+export type ToHtmlTransformation = (node: DocumentationNode, context: ToHtmlContext) => Nodes;
+
+// @alpha
+export interface ToHtmlTransformations {
+    [documentationNodeKind: string]: ToHtmlTransformation;
+}
+
 // @public
 export type TransformApiItemWithChildren<TApiItem extends ApiItem> = (apiItem: TApiItem, config: Required<ApiItemTransformationConfiguration>, generateChildSection: (apiItem: ApiItem) => SectionNode[]) => SectionNode[];
 

tools/api-markdown-documenter/package.json

@@ -75,7 +75,9 @@
 		"@microsoft/api-extractor-model": "~7.28.2",
 		"@microsoft/tsdoc": "^0.14.2",
 		"@rushstack/node-core-library": "^3.55.2",
-		"chalk": "^4.1.2"
+		"chalk": "^4.1.2",
+		"hast-util-raw": "^9.0.2",
+		"hastscript": "^9.0.0"
 	},
 	"devDependencies": {
 		"@fluid-tools/markdown-magic": "file:../markdown-magic",
@@ -85,6 +87,7 @@
 		"@fluidframework/mocha-test-setup": "~2.0.0-internal.6.2.0",
 		"@microsoft/api-extractor": "^7.42.3",
 		"@types/chai": "^4.3.4",
+		"@types/hast": "^3.0.4",
 		"@types/mocha": "^10.0.1",
 		"@types/node": "^18.15.11",
 		"@types/unist": "^2.0.6",