feat: load TOC

packages/myst-cli/package.json

@@ -82,6 +82,7 @@
     "myst-spec": "^0.0.5",
     "myst-spec-ext": "^1.4.0",
     "myst-templates": "^1.0.18",
+    "myst-toc": "^0.0.0",
     "myst-to-docx": "^1.0.10",
     "myst-to-jats": "^1.0.26",
     "myst-to-md": "^1.0.11",

packages/myst-cli/src/build/utils/collectExportOptions.ts

@@ -11,8 +11,9 @@ import {
 import { VFile } from 'vfile';
 import { findCurrentProjectAndLoad } from '../../config.js';
 import { logMessagesFromVFile } from '../../utils/logging.js';
-import { validateTOC } from '../../utils/toc.js';
-import { projectFromTOC } from '../../project/fromTOC.js';
+import { validateSphinxTOC } from '../../utils/toc.js';
+import { projectFromTOC, projectFromSphinxTOC } from '../../project/fromTOC.js';
+
 import { loadProjectFromDisk } from '../../project/load.js';
 import type { LocalProject } from '../../project/types.js';
 import type { ISession } from '../../session/types.js';
@@ -64,6 +65,21 @@ export function resolveArticlesFromProject(
 }
 
 function resolveArticlesFromTOC(
+  session: ISession,
+  exp: ExportWithFormat,
+  path: string,
+  vfile: VFile,
+): ResolvedArticles {
+  const allowLevelLessThanOne = [
+    ExportFormats.tex,
+    ExportFormats.pdf,
+    ExportFormats.pdftex,
+  ].includes(exp.format);
+  const proj = projectFromTOC(session, path, exp.toc!, allowLevelLessThanOne ? -1 : 1);
+  return resolveArticlesFromProject(exp, proj, vfile);
+}
+
+function resolveArticlesFromSphinxTOC(
   session: ISession,
   exp: ExportWithFormat,
   tocPath: string,
@@ -74,7 +90,7 @@ function resolveArticlesFromTOC(
     ExportFormats.pdf,
     ExportFormats.pdftex,
   ].includes(exp.format);
-  const proj = projectFromTOC(session, tocPath, allowLevelLessThanOne ? -1 : 1);
+  const proj = projectFromSphinxTOC(session, tocPath, allowLevelLessThanOne ? -1 : 1);
   return resolveArticlesFromProject(exp, proj, vfile);
 }
 
@@ -159,10 +175,7 @@ export function resolveArticles(
   let resolved: ResolvedArticles = { articles, sub_articles };
   // First, respect explicit toc. If articles/sub_articles are already defined, toc is ignored.
   if (exp.toc && !resolved.articles && !resolved.sub_articles) {
-    const resolvedTOC = path.resolve(path.dirname(sourceFile), exp.toc);
-    if (validateTOC(session, resolvedTOC)) {
-      resolved = resolveArticlesFromTOC(session, exp, resolvedTOC, vfile);
-    }
+    resolved = resolveArticlesFromTOC(session, exp, projectPath ?? '.', vfile);
   }
   // If no articles are specified, use the sourceFile for article
   if (!resolved.articles && SOURCE_EXTENSIONS.includes(path.extname(sourceFile))) {
@@ -174,8 +187,8 @@ export function resolveArticles(
   }
   // If still no articles, try to use explicit or implicit project toc
   if (!resolved.articles && !resolved.sub_articles) {
-    if (validateTOC(session, projectPath ?? '.')) {
-      resolved = resolveArticlesFromTOC(session, exp, projectPath ?? '.', vfile);
+    if (validateSphinxTOC(session, projectPath ?? '.')) {
+      resolved = resolveArticlesFromSphinxTOC(session, exp, projectPath ?? '.', vfile);
     } else {
       const cachedProject = selectors.selectLocalProject(
         session.store.getState(),

packages/myst-cli/src/project/fromPath.ts

@@ -10,7 +10,7 @@ import { fileInfo } from '../utils/fileInfo.js';
 import { nextLevel } from '../utils/nextLevel.js';
 import { VALID_FILE_EXTENSIONS, isValidFile } from '../utils/resolveExtension.js';
 import { shouldIgnoreFile } from '../utils/shouldIgnoreFile.js';
-import { pagesFromTOC } from './fromTOC.js';
+import { pagesFromSphinxTOC } from './fromTOC.js';
 import type {
   PageLevels,
   LocalProjectFolder,
@@ -60,7 +60,7 @@ function projectPagesFromPath(
   if (contents.includes(join(path, '_toc.yml'))) {
     const prevLevel = (level < 2 ? 1 : level - 1) as PageLevels;
     try {
-      return pagesFromTOC(session, path, prevLevel);
+      return pagesFromSphinxTOC(session, path, prevLevel);
     } catch {
       if (!suppressWarnings) {
         addWarningForFile(

packages/myst-cli/src/project/fromTOC.ts

@@ -1,9 +1,10 @@
 import fs from 'node:fs';
+import assert from 'node:assert';
 import { join, parse } from 'node:path';
 import { RuleId } from 'myst-common';
 import type { ISession } from '../session/types.js';
 import type { JupyterBookChapter } from '../utils/toc.js';
-import { readTOC, tocFile } from '../utils/toc.js';
+import { readSphinxTOC, tocFile } from '../utils/toc.js';
 import { VALID_FILE_EXTENSIONS, resolveExtension } from '../utils/resolveExtension.js';
 import { fileInfo } from '../utils/fileInfo.js';
 import { addWarningForFile } from '../utils/addWarningForFile.js';
@@ -15,8 +16,83 @@ import type {
   LocalProject,
   PageSlugs,
 } from './types.js';
+import type { TOC, Entry, ParentEntry } from 'myst-toc';
+import { isFile, isPattern, isURL } from 'myst-toc';
 
-function pagesFromChapters(
+function pagesFromEntries(
+  session: ISession,
+  path: string,
+  entries: Entry[],
+  pages: (LocalProjectFolder | LocalProjectPage)[] = [],
+  level: PageLevels = 1,
+  pageSlugs: PageSlugs,
+): (LocalProjectFolder | LocalProjectPage)[] {
+  for (const entry of entries) {
+    if (isFile(entry)) {
+      const file = resolveExtension(join(path, entry.file));
+      if (file) {
+        const { slug } = fileInfo(file, pageSlugs);
+        pages.push({ file, level, slug });
+      } else {
+        addWarningForFile(session, path, `Referenced file not found: ${entry.file}`, 'error', {
+          ruleId: RuleId.tocContentsExist,
+        });
+      }
+    } else if (isPattern(entry)) {
+      throw new Error('Not implemented!');
+    } else if (isURL(entry)) {
+      throw new Error('Not implemented!');
+    } else {
+      // Parent Entry
+      pages.push({ level, title: entry.title });
+    }
+
+    // Do we have any children?
+    const parentEntry = entry as Partial<ParentEntry>;
+    if (parentEntry.children) {
+      pagesFromEntries(session, path, parentEntry.children, pages, nextLevel(level), pageSlugs);
+    }
+  }
+  return pages;
+}
+
+/**
+ * Build project structure from a MyST TOC
+ *
+ * Starting level may be provided; by default this is 1. Numbers up to
+ * 6 may be provided for pages to start at a lower level. Level may
+ * also be -1 or 0. In these cases, the first "part" level will be -1
+ * and the first "chapter" level will be 0; However, "sections"
+ * will never be level < 1.
+ */
+export function projectFromTOC(
+  session: ISession,
+  path: string,
+  toc: TOC,
+  level: PageLevels = 1,
+): Omit<LocalProject, 'bibliography'> {
+  const pageSlugs: PageSlugs = {};
+  const [root, ...entries] = toc;
+  assert(isFile(root));
+  const indexFile = resolveExtension(join(path, root.file));
+  if (!indexFile) {
+    throw Error(
+      `The table of contents could not find file "${
+        root.file
+      }" defined as the first (root) page. Please ensure that one of these files is defined:\n- ${VALID_FILE_EXTENSIONS.map(
+        (ext) => join(path, `${root.file}${ext}`),
+      ).join('\n- ')}\n`,
+    );
+  }
+  const { slug } = fileInfo(indexFile, pageSlugs);
+  const pages: (LocalProjectFolder | LocalProjectPage)[] = [];
+  // Do not allow parts to have level < -1
+  if (level < -1) level = -1;
+  pagesFromEntries(session, path, entries, pages, level, pageSlugs);
+  return { path: path || '.', file: indexFile, index: slug, pages };
+}
+
+function pagesFromSphinxChapters(
   session: ISession,
   path: string,
   chapters: JupyterBookChapter[],
@@ -46,7 +122,7 @@ function pagesFromChapters(
       pages.push({ level, title: chapter.title });
     }
     if (chapter.sections) {
-      pagesFromChapters(session, path, chapter.sections, pages, nextLevel(level), pageSlugs);
+      pagesFromSphinxChapters(session, path, chapter.sections, pages, nextLevel(level), pageSlugs);
     }
   });
   return pages;
@@ -61,7 +137,7 @@ function pagesFromChapters(
  * and the first "chapter" level will be 0; However, "sections"
  * will never be level < 1.
  */
-export function projectFromTOC(
+export function projectFromSphinxTOC(
   session: ISession,
   path: string,
   level: PageLevels = 1,
@@ -71,7 +147,11 @@ export function projectFromTOC(
     throw new Error(`Could not find TOC "${filename}". Please create a '_toc.yml'.`);
   }
   const { dir, base } = parse(filename);
-  const toc = readTOC(session.log, { filename: base, path: dir });
+  const toc = readSphinxTOC(session.log, { filename: base, path: dir });
+
+  addWarningForFile(session, filename, `Encountered legacy TOC: ${filename}`, 'warn', {
+    ruleId: RuleId.encounteredLegacyTOC,
+  });
   const pageSlugs: PageSlugs = {};
   const indexFile = resolveExtension(join(dir, toc.root));
   if (!indexFile) {
@@ -88,11 +168,11 @@ export function projectFromTOC(
   if (toc.sections) {
     // Do not allow sections to have level < 1
     if (level < 1) level = 1;
-    pagesFromChapters(session, path, toc.sections, pages, level, pageSlugs);
+    pagesFromSphinxChapters(session, path, toc.sections, pages, level, pageSlugs);
   } else if (toc.chapters) {
     // Do not allow chapters to have level < 0
     if (level < 0) level = 0;
-    pagesFromChapters(session, path, toc.chapters, pages, level, pageSlugs);
+    pagesFromSphinxChapters(session, path, toc.chapters, pages, level, pageSlugs);
   } else if (toc.parts) {
     // Do not allow parts to have level < -1
     if (level < -1) level = -1;
@@ -101,24 +181,40 @@ export function projectFromTOC(
         pages.push({ title: part.caption || `Part ${index + 1}`, level });
       }
       if (part.chapters) {
-        pagesFromChapters(session, path, part.chapters, pages, nextLevel(level), pageSlugs);
+        pagesFromSphinxChapters(session, path, part.chapters, pages, nextLevel(level), pageSlugs);
       }
     });
   }
   return { path: dir || '.', file: indexFile, index: slug, pages };
 }
 
 /**
- * Return only project pages/folders from a '_toc.yml' file
+ * Return only project pages/folders from a TOC
  *
  * The root file is converted into just another top-level page.
  */
 export function pagesFromTOC(
+  session: ISession,
+  path: string,
+  toc: TOC,
+  level: PageLevels,
+): (LocalProjectFolder | LocalProjectPage)[] {
+  const { file, index, pages } = projectFromTOC(session, path, toc, nextLevel(level));
+  pages.unshift({ file, slug: index, level });
+  return pages;
+}
+
+/**
+ * Return only project pages/folders from a '_toc.yml' file
+ *
+ * The root file is converted into just another top-level page.
+ */
+export function pagesFromSphinxTOC(
   session: ISession,
   path: string,
   level: PageLevels,
 ): (LocalProjectFolder | LocalProjectPage)[] {
-  const { file, index, pages } = projectFromTOC(session, path, nextLevel(level));
+  const { file, index, pages } = projectFromSphinxTOC(session, path, nextLevel(level));
   pages.unshift({ file, slug: index, level });
   return pages;
 }

packages/myst-cli/src/project/load.ts

@@ -10,10 +10,10 @@ import { selectors } from '../store/index.js';
 import { projects } from '../store/reducers.js';
 import { addWarningForFile } from '../utils/addWarningForFile.js';
 import { getAllBibTexFilesOnPath } from '../utils/getAllBibtexFiles.js';
-import { validateTOC } from '../utils/toc.js';
+import { validateSphinxTOC } from '../utils/toc.js';
 import { projectFromPath } from './fromPath.js';
-import { projectFromTOC } from './fromTOC.js';
 import { writeTOCFromProject } from './toTOC.js';
+import { projectFromTOC, projectFromSphinxTOC } from './fromTOC.js';
 import type { LocalProject, LocalProjectPage } from './types.js';
 
 /**
@@ -52,10 +52,13 @@ export async function loadProjectFromDisk(
   let newProject: Omit<LocalProject, 'bibliography'> | undefined;
   let { index, writeTOC } = opts || {};
   const projectConfigFile = selectors.selectLocalConfigFile(session.store.getState(), path);
-  if (validateTOC(session, path)) {
-    newProject = projectFromTOC(session, path);
+  // Legacy validator
+  if (validateSphinxTOC(session, path)) {
+    newProject = projectFromSphinxTOC(session, path);
     if (writeTOC) session.log.warn('Not writing the table of contents, it already exists!');
     writeTOC = false;
+  } else if (projectConfig?.toc !== undefined) {
+    newProject = projectFromTOC(session, path, projectConfig.toc);
   } else {
     const project = selectors.selectLocalProject(session.store.getState(), path);
     if (!index && !project?.implicitIndex && project?.file) {
@@ -68,22 +71,7 @@ export async function loadProjectFromDisk(
     throw new Error(`Could not load project from ${path}`);
   }
   if (writeTOC) {
-    try {
-      session.log.info(
-        `📓 Writing '_toc.yml' file to ${path === '.' ? 'the current directory' : path}`,
-      );
-      writeTOCFromProject(newProject, path);
-      // Re-load from TOC just in case there are subtle differences with resulting project
-      newProject = projectFromTOC(session, path);
-    } catch {
-      addWarningForFile(
-        session,
-        projectConfigFile,
-        `Error writing '_toc.yml' file to ${path}`,
-        'error',
-        { ruleId: RuleId.tocWritten },
-      );
-    }
+    throw new Error('Not implemented');
   }
   const allBibFiles = getAllBibTexFilesOnPath(session, path);
   let bibliography: string[];

packages/myst-cli/src/project/toc.spec.ts

@@ -2,7 +2,7 @@ import { describe, expect, it, beforeEach, vi } from 'vitest';
 import memfs from 'memfs';
 import { Session } from '../session';
 import { projectFromPath } from './fromPath';
-import { pagesFromTOC, projectFromTOC } from './fromTOC';
+import { pagesFromSphinxTOC, projectFromSphinxTOC } from './fromTOC';
 import { tocFromProject } from './toTOC';
 import { findProjectsOnPath } from './load';
 
@@ -627,7 +627,7 @@ chapters:
       - file: c
 `;
 
-describe('pagesFromToc', () => {
+describe('pagesFromSphinxTOC', () => {
   it('pages from toc', async () => {
     memfs.vol.fromJSON({
       '_toc.yml': TOC_FILE,
@@ -636,7 +636,7 @@ describe('pagesFromToc', () => {
       'b.md': '',
       'c.md': '',
     });
-    expect(pagesFromTOC(session, '.', 1)).toEqual([
+    expect(pagesFromSphinxTOC(session, '.', 1)).toEqual([
       { slug: 'index', file: 'index.md', level: 1 },
       { slug: 'a', file: 'a.md', level: 2 },
       { title: 'Sections', level: 2 },
@@ -652,7 +652,7 @@ describe('pagesFromToc', () => {
       'b.md': '',
       'c.md': '',
     });
-    expect(projectFromTOC(session, '.', 1)).toEqual({
+    expect(projectFromSphinxTOC(session, '.', 1)).toEqual({
       index: 'index',
       file: 'index.md',
       path: '.',
@@ -672,7 +672,7 @@ describe('pagesFromToc', () => {
       'b.md': '',
       'c.md': '',
     });
-    expect(projectFromTOC(session, '.', 0)).toEqual({
+    expect(projectFromSphinxTOC(session, '.', 0)).toEqual({
       index: 'index',
       file: 'index.md',
       path: '.',
@@ -692,7 +692,7 @@ describe('pagesFromToc', () => {
       'b.md': '',
       'c.md': '',
     });
-    expect(projectFromTOC(session, '.', -1)).toEqual({
+    expect(projectFromSphinxTOC(session, '.', -1)).toEqual({
       index: 'index',
       file: 'index.md',
       path: '.',
@@ -714,7 +714,7 @@ describe('pagesFromToc', () => {
       'd.md': '',
       'e.md': '',
     });
-    expect(pagesFromTOC(session, '.', 1)).toEqual([
+    expect(pagesFromSphinxTOC(session, '.', 1)).toEqual([
       { slug: 'index', file: 'index.md', level: 1 },
       { slug: 'a', file: 'a.md', level: 2 },
       { title: 'Sections', level: 2 },