From 63a958245261680ee72f1aca35cd343ff50a6305 Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Tue, 21 Jan 2025 09:29:15 -0700 Subject: [PATCH 1/7] =?UTF-8?q?=E2=9C=A8=20New=20TOC=20directive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/brave-dragons-pump.md | 7 +++ packages/myst-cli/src/process/mdast.ts | 10 ++++ packages/myst-directives/src/index.ts | 3 ++ packages/myst-directives/src/toc.ts | 32 ++++++++++++ packages/myst-transforms/src/index.ts | 1 + packages/myst-transforms/src/toc.ts | 72 ++++++++++++++++++++++++++ 6 files changed, 125 insertions(+) create mode 100644 .changeset/brave-dragons-pump.md create mode 100644 packages/myst-directives/src/toc.ts create mode 100644 packages/myst-transforms/src/toc.ts diff --git a/.changeset/brave-dragons-pump.md b/.changeset/brave-dragons-pump.md new file mode 100644 index 000000000..ac3276d79 --- /dev/null +++ b/.changeset/brave-dragons-pump.md @@ -0,0 +1,7 @@ +--- +'myst-directives': patch +'myst-transforms': patch +'myst-cli': patch +--- + +New TOC directive diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index 39acbdf2a..ce4046c1e 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -29,6 +29,7 @@ import { inlineMathSimplificationPlugin, checkLinkTextTransform, indexIdentifierPlugin, + buildTocTransform, } from 'myst-transforms'; import { unified } from 'unified'; import { select, selectAll } from 'unist-util-select'; @@ -76,6 +77,7 @@ import { kernelExecutionTransform, LocalDiskCache } from 'myst-execute'; import type { IOutput } from '@jupyterlab/nbformat'; import { rawDirectiveTransform } from '../transforms/raw.js'; import { addEditUrl } from '../utils/addEditUrl.js'; +import { localToManifestProject } from '../build/index.js'; const LINKS_SELECTOR = 'link,card,linkBlock'; @@ -306,6 +308,14 @@ export async function postProcessMdast( const { mdast, dependencies, frontmatter } = mdastPost; const state = new MultiPageReferenceResolver(pageReferenceStates, file, vfile); const externalReferences = Object.values(cache.$externalReferences); + const storeState = session.store.getState(); + const projectPath = selectors.selectCurrentProjectPath(storeState); + if (projectPath) { + const siteConfig = selectors.selectCurrentSiteConfig(storeState); + const projectSlug = siteConfig?.projects?.find((proj) => proj.path === projectPath)?.slug; + const manifestProject = await localToManifestProject(session, projectPath, projectSlug); + if (manifestProject) buildTocTransform(mdast, vfile, manifestProject?.pages, projectSlug); + } // NOTE: This is doing things in place, we should potentially make this a different state? const transformers = [ ...(extraLinkTransformers || []), diff --git a/packages/myst-directives/src/index.ts b/packages/myst-directives/src/index.ts index f3766426d..5a153e543 100644 --- a/packages/myst-directives/src/index.ts +++ b/packages/myst-directives/src/index.ts @@ -18,6 +18,7 @@ import { mystdemoDirective } from './mystdemo.js'; import { blockquoteDirective } from './blockquote.js'; import { rawDirective, rawLatexDirective, rawTypstDirective } from './raw.js'; import { divDirective } from './div.js'; +import { tocDirective } from './toc.js'; export const defaultDirectives = [ admonitionDirective, @@ -46,6 +47,7 @@ export const defaultDirectives = [ rawLatexDirective, rawTypstDirective, divDirective, + tocDirective, ]; export * from './utils.js'; @@ -68,3 +70,4 @@ export { mystdemoDirective } from './mystdemo.js'; export { blockquoteDirective } from './blockquote.js'; export { rawDirective, rawLatexDirective, rawTypstDirective } from './raw.js'; export { divDirective } from './div.js'; +export { tocDirective } from './toc.js'; diff --git a/packages/myst-directives/src/toc.ts b/packages/myst-directives/src/toc.ts new file mode 100644 index 000000000..663cb417b --- /dev/null +++ b/packages/myst-directives/src/toc.ts @@ -0,0 +1,32 @@ +import type { DirectiveSpec, DirectiveData, GenericNode } from 'myst-common'; +import { addCommonDirectiveOptions, commonDirectiveOptions } from './utils.js'; + +export const tocDirective: DirectiveSpec = { + name: 'toc', + alias: ['tableofcontents', 'table-of-contents'], + arg: { + type: 'myst', + doc: 'Heading to be included with table of contents', + }, + options: { + ...commonDirectiveOptions('toc'), + }, + run(data: DirectiveData): GenericNode[] { + const children: GenericNode[] = []; + if (data.arg) { + const parsedArg = data.arg as GenericNode[]; + if (parsedArg[0]?.type === 'heading') { + children.push(...parsedArg); + } else { + children.push({ + type: 'heading', + depth: 1, + children: parsedArg, + }); + } + } + const toc = { type: 'toc', children }; + addCommonDirectiveOptions(data, toc); + return [toc]; + }, +}; diff --git a/packages/myst-transforms/src/index.ts b/packages/myst-transforms/src/index.ts index 0d3f40d02..17e57b6f9 100644 --- a/packages/myst-transforms/src/index.ts +++ b/packages/myst-transforms/src/index.ts @@ -59,6 +59,7 @@ export { abbreviationPlugin, abbreviationTransform } from './abbreviations.js'; export { includeDirectivePlugin, includeDirectiveTransform } from './include.js'; export { containerChildrenPlugin, containerChildrenTransform } from './containers.js'; export { headingDepthPlugin, headingDepthTransform } from './headings.js'; +export { buildTocTransform } from './toc.js'; // Enumeration export type { IReferenceStateResolver, ReferenceKind, TargetCounts } from './enumerate.js'; diff --git a/packages/myst-transforms/src/toc.ts b/packages/myst-transforms/src/toc.ts new file mode 100644 index 000000000..924bb594c --- /dev/null +++ b/packages/myst-transforms/src/toc.ts @@ -0,0 +1,72 @@ +import { fileWarn, type GenericNode, type GenericParent } from 'myst-common'; +import type { List, Text } from 'myst-spec'; +import type { Link, ListItem } from 'myst-spec-ext'; +import { selectAll } from 'unist-util-select'; +import type { VFile } from 'vfile'; + +type ProjectPage = { + title: string; + level: number; + slug?: string; + enumerator?: string; +}; + +function listFromPages(pages: ProjectPage[], projectSlug?: string): List { + if (pages.length === 0) return { type: 'list', children: [] }; + let ignore = false; + const level = pages[0].level; + const children = pages + .map((page, index) => { + if (ignore) return undefined; + if (page.level < level) ignore = true; + if (page.level !== level) return undefined; + return listItemFromPages(pages.slice(index), projectSlug); + }) + .filter((item): item is ListItem => !!item); + return { type: 'list', children }; +} + +function listItemFromPages(pages: ProjectPage[], projectSlug?: string) { + if (pages.length === 0) return; + const { title, slug, enumerator, level } = pages[0]; + const text: Text = { + type: 'text', + value: `${enumerator ? `${enumerator} ` : ''}${title}`, + }; + const child: Text | Link = slug + ? ({ + type: 'link', + url: `${projectSlug ? `/${projectSlug}` : ''}/${slug}`, + internal: true, + children: [text], + } as Link) + : text; + const item: ListItem = { + type: 'listItem', + children: [child], + }; + if (pages[1] && pages[1].level > level) { + item.children.push(listFromPages(pages.slice(1), projectSlug)); + } + return item; +} + +export function buildTocTransform( + mdast: GenericParent, + vfile: VFile, + pages: ProjectPage[], + projectSlug?: string, +) { + if (pages.length === 0) return; + const tocs = selectAll('toc', mdast) as GenericNode[]; + if (!tocs.length) return; + if (pages[0].level !== 1) { + fileWarn(vfile, `First page of Table of Contents must be level 1`); + } + tocs.forEach((toc) => { + toc.type = 'block'; + toc.data = { part: 'toc' }; + if (!toc.children) toc.children = []; + toc.children.push(listFromPages(pages, projectSlug)); + }); +} From 44ddc3673e0b1278a515b8365f0bb32a0d4ce57b Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Thu, 23 Jan 2025 00:21:52 -0700 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=94=A7=20Ad=20toctree=20as=20toc=20di?= =?UTF-8?q?rective=20alias?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-directives/src/toc.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/myst-directives/src/toc.ts b/packages/myst-directives/src/toc.ts index 663cb417b..fc32d0ecb 100644 --- a/packages/myst-directives/src/toc.ts +++ b/packages/myst-directives/src/toc.ts @@ -3,7 +3,7 @@ import { addCommonDirectiveOptions, commonDirectiveOptions } from './utils.js'; export const tocDirective: DirectiveSpec = { name: 'toc', - alias: ['tableofcontents', 'table-of-contents'], + alias: ['tableofcontents', 'table-of-contents', 'toctree'], arg: { type: 'myst', doc: 'Heading to be included with table of contents', From 6404716f064f6d168104cb9ec3b6a65f9f6cc0ac Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Mon, 27 Jan 2025 19:38:10 -0700 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=94=A7=20Only=20run=20toc=20transform?= =?UTF-8?q?=20for=20site=20builds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-cli/src/process/mdast.ts | 27 +++++++++++++++++++++----- packages/myst-cli/src/process/site.ts | 2 ++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index ce4046c1e..459033aa0 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -291,11 +291,13 @@ export async function postProcessMdast( checkLinks, pageReferenceStates, extraLinkTransformers, + site, }: { file: string; checkLinks?: boolean; pageReferenceStates: ReferenceState[]; extraLinkTransformers?: LinkTransformer[]; + site?: boolean; }, ) { const toc = tic(); @@ -310,11 +312,26 @@ export async function postProcessMdast( const externalReferences = Object.values(cache.$externalReferences); const storeState = session.store.getState(); const projectPath = selectors.selectCurrentProjectPath(storeState); - if (projectPath) { - const siteConfig = selectors.selectCurrentSiteConfig(storeState); - const projectSlug = siteConfig?.projects?.find((proj) => proj.path === projectPath)?.slug; - const manifestProject = await localToManifestProject(session, projectPath, projectSlug); - if (manifestProject) buildTocTransform(mdast, vfile, manifestProject?.pages, projectSlug); + const siteConfig = selectors.selectCurrentSiteConfig(storeState); + const projectSlug = siteConfig?.projects?.find((proj) => proj.path === projectPath)?.slug; + const manifestProject = await localToManifestProject(session, projectPath, projectSlug); + if (site) { + buildTocTransform( + mdast, + vfile, + manifestProject + ? [ + { + title: manifestProject.title, + level: 1, + slug: '', + enumerator: manifestProject.enumerator, + }, + ...manifestProject.pages, + ] + : undefined, + projectSlug, + ); } // NOTE: This is doing things in place, we should potentially make this a different state? const transformers = [ diff --git a/packages/myst-cli/src/process/site.ts b/packages/myst-cli/src/process/site.ts index 8cc158c42..8a787b66c 100644 --- a/packages/myst-cli/src/process/site.ts +++ b/packages/myst-cli/src/process/site.ts @@ -487,6 +487,7 @@ export async function fastProcessFile( file: f, pageReferenceStates, extraLinkTransformers, + site: true, }); }), ); @@ -601,6 +602,7 @@ export async function processProject( checkLinks: checkLinks || strict, pageReferenceStates, extraLinkTransformers, + site: true, }), ), ); From 782e8a3dcfdaa92e51b47977d4dbf67d8b99c919 Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Mon, 27 Jan 2025 19:38:40 -0700 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=94=A7=20Support=20page=20and=20secti?= =?UTF-8?q?on=20tocs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-directives/src/toc.ts | 38 +- packages/myst-transforms/src/enumerate.ts | 4 +- packages/myst-transforms/src/toc.spec.ts | 413 ++++++++++++++++++++++ packages/myst-transforms/src/toc.ts | 141 +++++++- 4 files changed, 572 insertions(+), 24 deletions(-) create mode 100644 packages/myst-transforms/src/toc.spec.ts diff --git a/packages/myst-directives/src/toc.ts b/packages/myst-directives/src/toc.ts index fc32d0ecb..47ff35fdb 100644 --- a/packages/myst-directives/src/toc.ts +++ b/packages/myst-directives/src/toc.ts @@ -1,17 +1,44 @@ -import type { DirectiveSpec, DirectiveData, GenericNode } from 'myst-common'; +import type { VFile } from 'vfile'; +import { type DirectiveSpec, type DirectiveData, type GenericNode, fileError } from 'myst-common'; import { addCommonDirectiveOptions, commonDirectiveOptions } from './utils.js'; +const CONTEXTS = ['project', 'page', 'section']; + export const tocDirective: DirectiveSpec = { name: 'toc', - alias: ['tableofcontents', 'table-of-contents', 'toctree'], + alias: ['tableofcontents', 'table-of-contents', 'toctree', 'contents'], arg: { type: 'myst', doc: 'Heading to be included with table of contents', }, options: { + context: { + type: String, + doc: 'Table of Contents context; one of project, page, or section', + alias: ['kind'], + }, + depth: { + type: Number, + doc: 'Number of levels to include in Table of Contents; by default, all levels will be included', + alias: ['maxdepth'], + }, ...commonDirectiveOptions('toc'), }, - run(data: DirectiveData): GenericNode[] { + run(data: DirectiveData, vfile: VFile): GenericNode[] { + let context = data.options?.context + ? (data.options.context as string) + : data.name === 'contents' + ? 'section' + : 'project'; + if (!CONTEXTS.includes(context)) { + fileError(vfile, `Unknown context for ${data.name} directive: ${context}`); + context = 'project'; + } + let depth = data.options?.depth as number | undefined; + if (depth != null && depth < 1) { + fileError(vfile, `Table of Contents 'depth' must be a number greater than 0`); + depth = undefined; + } const children: GenericNode[] = []; if (data.arg) { const parsedArg = data.arg as GenericNode[]; @@ -20,12 +47,13 @@ export const tocDirective: DirectiveSpec = { } else { children.push({ type: 'heading', - depth: 1, + depth: 2, + enumerated: false, children: parsedArg, }); } } - const toc = { type: 'toc', children }; + const toc = { type: 'toc', kind: context, depth, children }; addCommonDirectiveOptions(data, toc); return [toc]; }, diff --git a/packages/myst-transforms/src/enumerate.ts b/packages/myst-transforms/src/enumerate.ts index 037634487..946e07772 100644 --- a/packages/myst-transforms/src/enumerate.ts +++ b/packages/myst-transforms/src/enumerate.ts @@ -735,7 +735,8 @@ export function addContainerCaptionNumbersTransform( * Raise a warning if `target` linked by `node` has an implicit reference */ function implicitTargetWarning(target: Target, node: GenericNode, opts: StateResolverOptions) { - if ((target.node as GenericNode).implicit && opts.state.vfile) { + // suppressImplicitWarning is used, for example, in the table of contents directive + if ((target.node as GenericNode).implicit && opts.state.vfile && !node.suppressImplicitWarning) { fileWarn( opts.state.vfile, `Linking "${target.node.identifier}" to an implicit ${target.kind} reference, best practice is to create an explicit reference.`, @@ -747,6 +748,7 @@ function implicitTargetWarning(target: Target, node: GenericNode, opts: StateRes }, ); } + delete node.suppressImplicitWarning; } export const resolveReferenceLinksTransform = (tree: GenericParent, opts: StateResolverOptions) => { diff --git a/packages/myst-transforms/src/toc.spec.ts b/packages/myst-transforms/src/toc.spec.ts new file mode 100644 index 000000000..b043fe9f4 --- /dev/null +++ b/packages/myst-transforms/src/toc.spec.ts @@ -0,0 +1,413 @@ +import { describe, expect, test } from 'vitest'; +import { buildTocTransform } from './toc'; +import { VFile } from 'vfile'; +import { toText } from 'myst-common'; + +describe('Test toc transformation', () => { + test('Project Toc - basic', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [{ type: 'toc', kind: 'project', children: [] }], + } as any; + buildTocTransform(mdast, vfile, [ + { title: 'One', level: 1, slug: '' }, + { title: 'Two', level: 1, slug: 'two' }, + { title: 'Three', level: 2, slug: 'three' }, + { title: 'Four', level: 1, slug: 'four' }, + ]); + expect(mdast.children[0].type).toBe('block'); + expect(mdast.children[0].data.part).toBe('toc:project'); + expect(mdast.children[0].children[0].type).toBe('list'); + expect(mdast.children[0].children[0].children.length).toBe(3); + expect(mdast.children[0].children[0].children[0].children.length).toBe(1); + expect(mdast.children[0].children[0].children[1].children.length).toBe(2); + expect(mdast.children[0].children[0].children[0].children[0].url).toBe('/'); + expect(mdast.children[0].children[0].children[1].children[0].url).toBe('/two'); + expect(toText(mdast.children[0].children[0].children[1].children[0])).toBe('Two'); + }); + test('Project Toc - with project slug and enumerators', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [{ type: 'toc', kind: 'project', children: [] }], + } as any; + buildTocTransform( + mdast, + vfile, + [ + { title: 'One', level: 1, slug: '', enumerator: '1.1' }, + { title: 'Two', level: 1, slug: 'two', enumerator: '1.2' }, + { title: 'Three', level: 2, slug: 'three', enumerator: '1.2.1' }, + { title: 'Four', level: 1, slug: 'four', enumerator: '1.3' }, + ], + 'slug', + ); + expect(mdast.children[0].type).toBe('block'); + expect(mdast.children[0].data.part).toBe('toc:project'); + expect(mdast.children[0].children[0].type).toBe('list'); + expect(mdast.children[0].children[0].children.length).toBe(3); + expect(mdast.children[0].children[0].children[0].children.length).toBe(1); + expect(mdast.children[0].children[0].children[1].children.length).toBe(2); + expect(mdast.children[0].children[0].children[0].children[0].url).toBe('/slug/'); + expect(mdast.children[0].children[0].children[1].children[0].url).toBe('/slug/two'); + expect(toText(mdast.children[0].children[0].children[1].children[0])).toBe('1.2 Two'); + }); + test('Project Toc - no links', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [{ type: 'toc', kind: 'project', children: [] }], + } as any; + buildTocTransform(mdast, vfile, [ + { title: 'One', level: 1 }, + { title: 'Two', level: 1 }, + { title: 'Three', level: 2 }, + { title: 'Four', level: 1 }, + ]); + expect(mdast.children[0].type).toBe('block'); + expect(mdast.children[0].data.part).toBe('toc:project'); + expect(mdast.children[0].children[0].type).toBe('list'); + expect(mdast.children[0].children[0].children.length).toBe(3); + expect(mdast.children[0].children[0].children[0].children.length).toBe(1); + expect(mdast.children[0].children[0].children[1].children.length).toBe(2); + expect(mdast.children[0].children[0].children[0].children[0].url).toBeUndefined(); + expect(mdast.children[0].children[0].children[1].children[0].value).toBe('Two'); + }); + test('Project Toc - heading depth', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [{ type: 'toc', kind: 'project', depth: 1, children: [] }], + } as any; + buildTocTransform( + mdast, + vfile, + [ + { title: 'One', level: 1, slug: '', enumerator: '1.1' }, + { title: 'Two', level: 1, slug: 'two', enumerator: '1.2' }, + { title: 'Three', level: 2, slug: 'three', enumerator: '1.2.1' }, + { title: 'Four', level: 1, slug: 'four', enumerator: '1.3' }, + ], + 'slug', + ); + expect(mdast.children[0].type).toBe('block'); + expect(mdast.children[0].data.part).toBe('toc:project'); + expect(mdast.children[0].children[0].type).toBe('list'); + expect(mdast.children[0].children[0].children.length).toBe(3); + expect(mdast.children[0].children[0].children[0].children.length).toBe(1); + expect(mdast.children[0].children[0].children[1].children.length).toBe(1); + expect(mdast.children[0].children[0].children[0].children[0].url).toBe('/slug/'); + expect(mdast.children[0].children[0].children[1].children[0].url).toBe('/slug/two'); + expect(toText(mdast.children[0].children[0].children[1].children[0])).toBe('1.2 Two'); + }); + test('Page Toc - basic', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [ + { + type: 'heading', + children: [{ type: 'text', value: 'One' }], + depth: 1, + identifier: 'one', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Two' }], + depth: 1, + identifier: 'two', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Three' }], + depth: 2, + identifier: 'three', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Four' }], + depth: 1, + identifier: 'four', + }, + { type: 'toc', kind: 'page', children: [] }, + ], + } as any; + buildTocTransform(mdast, vfile); + expect(mdast.children[4].type).toBe('block'); + expect(mdast.children[4].data.part).toBe('toc:page'); + expect(mdast.children[4].children[0].type).toBe('list'); + expect(mdast.children[4].children[0].children.length).toBe(3); + expect(mdast.children[4].children[0].children[0].children.length).toBe(1); + expect(mdast.children[4].children[0].children[1].children.length).toBe(2); + expect(mdast.children[4].children[0].children[0].children[0].url).toBe('#one'); + expect(mdast.children[4].children[0].children[1].children[0].url).toBe('#two'); + expect(toText(mdast.children[4].children[0].children[1].children[0])).toBe('Two'); + }); + test('Page Toc - with enumerators', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [ + { + type: 'heading', + children: [{ type: 'text', value: 'One' }], + depth: 1, + identifier: 'one', + enumerator: '1.1', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Two' }], + depth: 1, + identifier: 'two', + enumerator: '1.2', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Three' }], + depth: 2, + identifier: 'three', + enumerator: '1.2.1', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Four' }], + depth: 1, + identifier: 'four', + enumerator: '1.3', + }, + { type: 'toc', kind: 'page', children: [] }, + ], + } as any; + buildTocTransform(mdast, vfile); + expect(mdast.children[4].type).toBe('block'); + expect(mdast.children[4].data.part).toBe('toc:page'); + expect(mdast.children[4].children[0].type).toBe('list'); + expect(mdast.children[4].children[0].children.length).toBe(3); + expect(mdast.children[4].children[0].children[0].children.length).toBe(1); + expect(mdast.children[4].children[0].children[1].children.length).toBe(2); + expect(mdast.children[4].children[0].children[0].children[0].url).toBe('#one'); + expect(mdast.children[4].children[0].children[1].children[0].url).toBe('#two'); + expect(toText(mdast.children[4].children[0].children[1].children[0])).toBe('1.2 Two'); + }); + test('Page Toc - no identifiers', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [ + { + type: 'heading', + children: [{ type: 'text', value: 'One' }], + depth: 1, + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Two' }], + depth: 1, + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Three' }], + depth: 2, + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Four' }], + depth: 1, + }, + { type: 'toc', kind: 'page', children: [] }, + ], + } as any; + buildTocTransform(mdast, vfile); + expect(mdast.children[4].type).toBe('block'); + expect(mdast.children[4].data.part).toBe('toc:page'); + expect(mdast.children[4].children[0].type).toBe('list'); + expect(mdast.children[4].children[0].children.length).toBe(3); + expect(mdast.children[4].children[0].children[0].children.length).toBe(1); + expect(mdast.children[4].children[0].children[1].children.length).toBe(2); + expect(mdast.children[4].children[0].children[0].children[0].url).toBeUndefined(); + expect(toText(mdast.children[4].children[0].children[1].children[0])).toBe('Two'); + }); + test('Page Toc - heading depth', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [ + { + type: 'heading', + children: [{ type: 'text', value: 'One' }], + depth: 1, + identifier: 'one', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Two' }], + depth: 1, + identifier: 'two', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Three' }], + depth: 2, + identifier: 'three', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Four' }], + depth: 1, + identifier: 'four', + }, + { type: 'toc', kind: 'page', depth: 1, children: [] }, + ], + } as any; + buildTocTransform(mdast, vfile); + expect(mdast.children[4].type).toBe('block'); + expect(mdast.children[4].data.part).toBe('toc:page'); + expect(mdast.children[4].children[0].type).toBe('list'); + expect(mdast.children[4].children[0].children.length).toBe(3); + expect(mdast.children[4].children[0].children[0].children.length).toBe(1); + expect(mdast.children[4].children[0].children[1].children.length).toBe(1); + expect(mdast.children[4].children[0].children[0].children[0].url).toBe('#one'); + expect(mdast.children[4].children[0].children[1].children[0].url).toBe('#two'); + expect(toText(mdast.children[4].children[0].children[1].children[0])).toBe('Two'); + }); + test('Section Toc - basic', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [ + { + type: 'heading', + children: [{ type: 'text', value: 'One' }], + depth: 1, + identifier: 'one', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Two' }], + depth: 1, + identifier: 'two', + }, + { type: 'toc', kind: 'section', children: [] }, + { + type: 'heading', + children: [{ type: 'text', value: 'Three' }], + depth: 2, + identifier: 'three', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Four' }], + depth: 1, + identifier: 'four', + }, + ], + } as any; + buildTocTransform(mdast, vfile); + expect(mdast.children[2].type).toBe('block'); + expect(mdast.children[2].data.part).toBe('toc:section'); + expect(mdast.children[2].children[0].type).toBe('list'); + expect(mdast.children[2].children[0].children.length).toBe(1); + expect(mdast.children[2].children[0].children[0].children.length).toBe(1); + expect(mdast.children[2].children[0].children[0].children[0].url).toBe('#three'); + expect(toText(mdast.children[2].children[0].children[0].children[0])).toBe('Three'); + }); + test('Section Toc - nested', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [ + { + type: 'heading', + children: [{ type: 'text', value: 'One' }], + depth: 1, + identifier: 'one', + }, + { type: 'toc', kind: 'section', children: [] }, + { + type: 'heading', + children: [{ type: 'text', value: 'Two' }], + depth: 1, + identifier: 'two', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Three' }], + depth: 2, + identifier: 'three', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Four' }], + depth: 1, + identifier: 'four', + }, + ], + } as any; + buildTocTransform(mdast, vfile); + expect(mdast.children[1].type).toBe('block'); + expect(mdast.children[1].data.part).toBe('toc:section'); + expect(mdast.children[1].children[0].type).toBe('list'); + expect(mdast.children[1].children[0].children.length).toBe(2); + expect(mdast.children[1].children[0].children[0].children.length).toBe(2); + expect(mdast.children[1].children[0].children[0].children[0].url).toBe('#two'); + expect(toText(mdast.children[1].children[0].children[0].children[0])).toBe('Two'); + expect(mdast.children[1].children[0].children[1].children[0].url).toBe('#four'); + }); + test('Section Toc - with heading', () => { + const vfile = new VFile(); + const mdast = { + type: 'root', + children: [ + { + type: 'heading', + children: [{ type: 'text', value: 'One' }], + depth: 1, + identifier: 'one', + }, + { + type: 'toc', + kind: 'section', + children: [ + { + type: 'heading', + children: [{ type: 'text', value: 'My ToC' }], + depth: 2, + identifier: 'my-toc', + }, + ], + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Two' }], + depth: 1, + identifier: 'two', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Three' }], + depth: 2, + identifier: 'three', + }, + { + type: 'heading', + children: [{ type: 'text', value: 'Four' }], + depth: 1, + identifier: 'four', + }, + ], + } as any; + buildTocTransform(mdast, vfile); + expect(mdast.children[1].type).toBe('block'); + expect(mdast.children[1].data.part).toBe('toc:section'); + expect(mdast.children[1].children[0].type).toBe('heading'); + expect(toText(mdast.children[1].children[0])).toBe('My ToC'); + expect(mdast.children[1].children[1].type).toBe('list'); + expect(mdast.children[1].children[1].children.length).toBe(2); + expect(mdast.children[1].children[1].children[0].children.length).toBe(2); + expect(mdast.children[1].children[1].children[0].children[0].url).toBe('#two'); + expect(toText(mdast.children[1].children[1].children[0].children[0])).toBe('Two'); + expect(mdast.children[1].children[1].children[1].children[0].url).toBe('#four'); + }); +}); diff --git a/packages/myst-transforms/src/toc.ts b/packages/myst-transforms/src/toc.ts index 924bb594c..1823a8863 100644 --- a/packages/myst-transforms/src/toc.ts +++ b/packages/myst-transforms/src/toc.ts @@ -1,6 +1,6 @@ -import { fileWarn, type GenericNode, type GenericParent } from 'myst-common'; +import { fileError, fileWarn, toText, type GenericNode, type GenericParent } from 'myst-common'; import type { List, Text } from 'myst-spec'; -import type { Link, ListItem } from 'myst-spec-ext'; +import type { Heading, Link, ListItem } from 'myst-spec-ext'; import { selectAll } from 'unist-util-select'; import type { VFile } from 'vfile'; @@ -33,20 +33,62 @@ function listItemFromPages(pages: ProjectPage[], projectSlug?: string) { type: 'text', value: `${enumerator ? `${enumerator} ` : ''}${title}`, }; - const child: Text | Link = slug + const child: Text | Link = + slug != null + ? ({ + type: 'link', + url: `${projectSlug ? `/${projectSlug}` : ''}/${slug}`, + internal: true, + children: [text], + } as Link) + : text; + const item: ListItem = { + type: 'listItem', + children: [child], + }; + if (pages[1] && pages[1].level > level) { + item.children.push(listFromPages(pages.slice(1), projectSlug)); + } + return item; +} + +function listFromHeadings(headings: Heading[]): List { + if (headings.length === 0) return { type: 'list', children: [] }; + let ignore = false; + const depth = headings[0].depth; + const children = headings + .map((heading, index) => { + if (ignore) return undefined; + if (heading.depth < depth) ignore = true; + if (heading.depth !== depth) return undefined; + return listItemFromHeadings(headings.slice(index)); + }) + .filter((item): item is ListItem => !!item); + return { type: 'list', children }; +} + +function listItemFromHeadings(headings: Heading[]) { + if (headings.length === 0) return; + const { children, enumerator, depth, identifier } = headings[0]; + const text: Text = { + type: 'text', + value: `${enumerator ? `${enumerator} ` : ''}${toText(children)}`, + }; + const child: Text | Link = identifier ? ({ type: 'link', - url: `${projectSlug ? `/${projectSlug}` : ''}/${slug}`, + url: `#${identifier}`, internal: true, children: [text], + suppressImplicitWarning: true, } as Link) : text; const item: ListItem = { type: 'listItem', children: [child], }; - if (pages[1] && pages[1].level > level) { - item.children.push(listFromPages(pages.slice(1), projectSlug)); + if (headings[1] && headings[1].depth > depth) { + item.children.push(listFromHeadings(headings.slice(1))); } return item; } @@ -54,19 +96,82 @@ function listItemFromPages(pages: ProjectPage[], projectSlug?: string) { export function buildTocTransform( mdast: GenericParent, vfile: VFile, - pages: ProjectPage[], + pages?: ProjectPage[], projectSlug?: string, ) { - if (pages.length === 0) return; - const tocs = selectAll('toc', mdast) as GenericNode[]; - if (!tocs.length) return; - if (pages[0].level !== 1) { - fileWarn(vfile, `First page of Table of Contents must be level 1`); + const tocHeadings = selectAll('toc > heading', mdast); + const tocsAndHeadings = selectAll('toc,heading', mdast).filter((item) => { + // Do not include toc headings anywhere in this transform + return !tocHeadings.includes(item); + }) as GenericNode[]; + if (!tocsAndHeadings.find((node) => node.type === 'toc')) return; + const projectTocs = tocsAndHeadings.filter( + (node) => node.type === 'toc' && node.kind === 'project', + ); + const pageTocs = tocsAndHeadings.filter((node) => node.type === 'toc' && node.kind === 'page'); + const sectionTocs = tocsAndHeadings.filter( + (node) => node.type === 'toc' && node.kind === 'section', + ); + if (projectTocs.length) { + if (!pages) { + fileError(vfile, `Pages not available to build Table of Contents`); + } else { + if (pages[0].level !== 1) { + fileWarn(vfile, `First page of Table of Contents must be level 1`); + } + projectTocs.forEach((toc) => { + const filteredPages = toc.depth ? pages.filter((page) => page.level <= toc.depth) : pages; + toc.type = 'block'; + delete toc.kind; + toc.data = { part: 'toc:project' }; + if (!toc.children) toc.children = []; + toc.children.push(listFromPages(filteredPages, projectSlug)); + }); + } + } + if (pageTocs.length) { + const headings = tocsAndHeadings.filter((node) => node.type === 'heading') as Heading[]; + if (headings.length === 0) { + fileWarn(vfile, `No page headings found for Table of Contents`); + } else { + if (Math.min(...headings.map((h) => h.depth)) !== headings[0].depth) { + fileWarn(vfile, 'Page heading levels do not start with highest level'); + } + pageTocs.forEach((toc) => { + const filteredHeadings = toc.depth + ? headings.filter((heading) => heading.depth - headings[0].depth < toc.depth) + : headings; + toc.type = 'block'; + delete toc.kind; + toc.data = { part: 'toc:page' }; + if (!toc.children) toc.children = []; + toc.children.push(listFromHeadings(filteredHeadings)); + }); + } + } + if (sectionTocs.length) { + tocsAndHeadings.forEach((toc, index) => { + if (toc.type !== 'toc' || toc.kind !== 'section') return; + const headings = tocsAndHeadings + .slice(index + 1) + .filter((h) => h.type === 'heading') as Heading[]; + if (headings.length === 0) { + fileWarn(vfile, `No section headings found for Table of Contents`); + } else { + const filteredHeadings = toc.depth + ? headings.filter((heading) => heading.depth - headings[0].depth < toc.depth) + : headings; + toc.type = 'block'; + delete toc.kind; + toc.data = { part: 'toc:section' }; + if (!toc.children) toc.children = []; + const nextSection = filteredHeadings.findIndex((h) => h.depth < filteredHeadings[0].depth); + toc.children.push( + listFromHeadings( + nextSection === -1 ? filteredHeadings : filteredHeadings.slice(0, nextSection), + ), + ); + } + }); } - tocs.forEach((toc) => { - toc.type = 'block'; - toc.data = { part: 'toc' }; - if (!toc.children) toc.children = []; - toc.children.push(listFromPages(pages, projectSlug)); - }); } From e306759799cd5b68598d2f7652bd03baee757978 Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Mon, 27 Jan 2025 19:39:34 -0700 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=94=A7=20Modify=20heading=20depth=20f?= =?UTF-8?q?or=20index=20heading?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-directives/src/indices.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/myst-directives/src/indices.ts b/packages/myst-directives/src/indices.ts index 7ec2ac225..6994f8375 100644 --- a/packages/myst-directives/src/indices.ts +++ b/packages/myst-directives/src/indices.ts @@ -95,7 +95,8 @@ export const genIndexDirective: DirectiveSpec = { } else { children.push({ type: 'heading', - depth: 1, + depth: 2, + enumerated: false, children: parsedArg, }); } From 57fff36c7f6323f9b73854839627fcd771c71e63 Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Mon, 27 Jan 2025 19:49:16 -0700 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=93=9A=20Add=20docs=20for=20toc=20dir?= =?UTF-8?q?ective?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/directives.md | 3 +++ packages/myst-directives/src/toc.ts | 1 + 2 files changed, 4 insertions(+) diff --git a/docs/directives.md b/docs/directives.md index 8d532d9f3..54bc11d09 100644 --- a/docs/directives.md +++ b/docs/directives.md @@ -62,3 +62,6 @@ description: A full list of the directives included in MyST Markdown by default. :::{myst:directive} table ::: + +:::{myst:directive} toc +::: diff --git a/packages/myst-directives/src/toc.ts b/packages/myst-directives/src/toc.ts index 47ff35fdb..7591f2905 100644 --- a/packages/myst-directives/src/toc.ts +++ b/packages/myst-directives/src/toc.ts @@ -6,6 +6,7 @@ const CONTEXTS = ['project', 'page', 'section']; export const tocDirective: DirectiveSpec = { name: 'toc', + doc: 'Inserts table of contents in the page. This may be for the project (each page has an entry), the current page (each heading has an entry), or the current section (only headings in the section have an entry).', alias: ['tableofcontents', 'table-of-contents', 'toctree', 'contents'], arg: { type: 'myst', From aa26dd97191c26d9a328c895053d0c1c7f20501a Mon Sep 17 00:00:00 2001 From: Franklin Koch Date: Tue, 28 Jan 2025 00:46:22 -0700 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=94=84=20Fix=20circular=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/myst-cli/src/build/site/manifest.ts | 54 +++----------- .../src/build/utils/projectManifest.ts | 73 +++++++++++++++++++ packages/myst-cli/src/process/mdast.ts | 15 ++-- 3 files changed, 92 insertions(+), 50 deletions(-) create mode 100644 packages/myst-cli/src/build/utils/projectManifest.ts diff --git a/packages/myst-cli/src/build/site/manifest.ts b/packages/myst-cli/src/build/site/manifest.ts index 356c0d3d8..ccc9f03a3 100644 --- a/packages/myst-cli/src/build/site/manifest.ts +++ b/packages/myst-cli/src/build/site/manifest.ts @@ -18,16 +18,18 @@ import type { RootState } from '../../store/index.js'; import { selectors } from '../../store/index.js'; import { transformBanner, transformThumbnail } from '../../transforms/images.js'; import { addWarningForFile } from '../../utils/addWarningForFile.js'; -import { fileTitle } from '../../utils/fileInfo.js'; import { resolveFrontmatterParts } from '../../utils/resolveFrontmatterParts.js'; import version from '../../version.js'; import { getSiteTemplate } from './template.js'; import { collectExportOptions } from '../utils/collectExportOptions.js'; import { filterPages } from '../../project/load.js'; import { getRawFrontmatterFromFile } from '../../process/file.js'; -import { castSession } from '../../session/cache.js'; - -type ManifestProject = Required['projects'][0]; +import type { ManifestProject } from '../utils/projectManifest.js'; +import { + indexFrontmatterFromProject, + manifestPagesFromProject, + manifestTitleFromProject, +} from '../utils/projectManifest.js'; export async function resolvePageExports(session: ISession, file: string): Promise { const exports = ( @@ -134,45 +136,9 @@ export async function localToManifestProject( const proj = selectors.selectLocalProject(state, projectPath); if (!proj) return null; // Update all of the page title to the frontmatter title - const { index, file: indexFile } = proj; + const { index } = proj; const projectFileInfo = selectors.selectFileInfo(state, proj.file); - const projectTitle = projConfig?.title || projectFileInfo.title || proj.index; - const cache = castSession(session); - const pages = await Promise.all( - proj.pages.map(async (page) => { - if ('file' in page) { - const fileInfo = selectors.selectFileInfo(state, page.file); - const title = fileInfo.title || fileTitle(page.file); - const short_title = fileInfo.short_title ?? undefined; - const description = fileInfo.description ?? ''; - const thumbnail = fileInfo.thumbnail ?? ''; - const thumbnailOptimized = fileInfo.thumbnailOptimized ?? ''; - const banner = fileInfo.banner ?? ''; - const bannerOptimized = fileInfo.bannerOptimized ?? ''; - const date = fileInfo.date ?? ''; - const tags = fileInfo.tags ?? []; - const { slug, level, file } = page; - const { frontmatter } = cache.$getMdast(file)?.post ?? {}; - const projectPage: ManifestProject['pages'][0] = { - slug, - title, - short_title, - description, - date, - thumbnail, - thumbnailOptimized, - banner, - bannerOptimized, - tags, - level, - enumerator: frontmatter?.enumerator, - }; - return projectPage; - } - return { ...page }; - }), - ); - + const pages = await manifestPagesFromProject(session, projectPath); const projFrontmatter = projConfig ? filterKeys(projConfig, PROJECT_FRONTMATTER_KEYS) : {}; const projConfigFile = selectors.selectLocalConfigFile(state, projectPath); const exports = projConfigFile ? await resolvePageExports(session, projConfigFile) : []; @@ -195,7 +161,7 @@ export async function localToManifestProject( session.publicPath(), { altOutputFolder: '/', webp: true }, ); - const { frontmatter } = cache.$getMdast(indexFile)?.post ?? {}; + const frontmatter = indexFrontmatterFromProject(session, projectPath); return { ...projFrontmatter, // TODO: a null in the project frontmatter should not fall back to index page @@ -214,7 +180,7 @@ export async function localToManifestProject( downloads, parts, bibliography: projFrontmatter.bibliography || [], - title: projectTitle || 'Untitled', + title: manifestTitleFromProject(session, projectPath), slug: projectSlug, index, enumerator: frontmatter?.enumerator, diff --git a/packages/myst-cli/src/build/utils/projectManifest.ts b/packages/myst-cli/src/build/utils/projectManifest.ts new file mode 100644 index 000000000..c06c15417 --- /dev/null +++ b/packages/myst-cli/src/build/utils/projectManifest.ts @@ -0,0 +1,73 @@ +import type { SiteManifest } from 'myst-config'; +import { castSession } from '../../session/cache.js'; +import type { ISession } from '../../session/types.js'; +import { selectors } from '../../store/index.js'; +import { fileTitle } from '../../utils/fileInfo.js'; +import type { PageFrontmatter } from 'myst-frontmatter'; + +export type ManifestProject = Required['projects'][0]; + +export async function manifestPagesFromProject(session: ISession, projectPath: string) { + const state = session.store.getState(); + const proj = selectors.selectLocalProject(state, projectPath); + if (!proj) return []; + const cache = castSession(session); + const pages = await Promise.all( + proj.pages.map(async (page) => { + if ('file' in page) { + const fileInfo = selectors.selectFileInfo(state, page.file); + const title = fileInfo.title || fileTitle(page.file); + const short_title = fileInfo.short_title ?? undefined; + const description = fileInfo.description ?? ''; + const thumbnail = fileInfo.thumbnail ?? ''; + const thumbnailOptimized = fileInfo.thumbnailOptimized ?? ''; + const banner = fileInfo.banner ?? ''; + const bannerOptimized = fileInfo.bannerOptimized ?? ''; + const date = fileInfo.date ?? ''; + const tags = fileInfo.tags ?? []; + const { slug, level, file } = page; + const { frontmatter } = cache.$getMdast(file)?.post ?? {}; + const projectPage: ManifestProject['pages'][0] = { + slug, + title, + short_title, + description, + date, + thumbnail, + thumbnailOptimized, + banner, + bannerOptimized, + tags, + level, + enumerator: frontmatter?.enumerator, + }; + return projectPage; + } + return { ...page }; + }), + ); + return pages; +} + +export function manifestTitleFromProject(session: ISession, projectPath: string) { + const state = session.store.getState(); + const projConfig = selectors.selectLocalProjectConfig(state, projectPath); + if (projConfig?.title) return projConfig.title; + const proj = selectors.selectLocalProject(state, projectPath); + if (!proj) return 'Untitled'; + const projectFileInfo = selectors.selectFileInfo(session.store.getState(), proj.file); + return projectFileInfo.title || proj.index || 'Untitled'; +} + +export function indexFrontmatterFromProject( + session: ISession, + projectPath: string, +): PageFrontmatter { + const state = session.store.getState(); + const cache = castSession(session); + const proj = selectors.selectLocalProject(state, projectPath); + if (!proj) return {}; + const { file } = proj; + const { frontmatter } = cache.$getMdast(file)?.post ?? {}; + return frontmatter ?? {}; +} diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index 459033aa0..53fbec47d 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -77,7 +77,11 @@ import { kernelExecutionTransform, LocalDiskCache } from 'myst-execute'; import type { IOutput } from '@jupyterlab/nbformat'; import { rawDirectiveTransform } from '../transforms/raw.js'; import { addEditUrl } from '../utils/addEditUrl.js'; -import { localToManifestProject } from '../build/index.js'; +import { + indexFrontmatterFromProject, + manifestPagesFromProject, + manifestTitleFromProject, +} from '../build/utils/projectManifest.js'; const LINKS_SELECTOR = 'link,card,linkBlock'; @@ -314,20 +318,19 @@ export async function postProcessMdast( const projectPath = selectors.selectCurrentProjectPath(storeState); const siteConfig = selectors.selectCurrentSiteConfig(storeState); const projectSlug = siteConfig?.projects?.find((proj) => proj.path === projectPath)?.slug; - const manifestProject = await localToManifestProject(session, projectPath, projectSlug); if (site) { buildTocTransform( mdast, vfile, - manifestProject + projectPath ? [ { - title: manifestProject.title, + title: manifestTitleFromProject(session, projectPath), level: 1, slug: '', - enumerator: manifestProject.enumerator, + enumerator: indexFrontmatterFromProject(session, projectPath).enumerator, }, - ...manifestProject.pages, + ...(await manifestPagesFromProject(session, projectPath)), ] : undefined, projectSlug,