🔄 Support recursive includes in myst and tex

.changeset/afraid-dots-look.md

@@ -0,0 +1,5 @@
+---
+'myst-to-typst': patch
+---
+
+Add newlines after typst include/embed

.changeset/dirty-steaks-wave.md

@@ -0,0 +1,6 @@
+---
+'myst-transforms': patch
+'myst-cli': patch
+---
+
+Support tex includes directly in myst processing

.changeset/funny-llamas-joke.md

@@ -0,0 +1,7 @@
+---
+'myst-transforms': patch
+'myst-cli': patch
+'mystmd': patch
+---
+
+Handle circular includes with nice errors and no infinite loops

.changeset/sour-dancers-float.md

@@ -0,0 +1,6 @@
+---
+'myst-transforms': patch
+'mystmd': patch
+---
+
+Revive basic recursive include

packages/myst-cli/src/process/file.ts

@@ -4,7 +4,6 @@ import { createHash } from 'node:crypto';
 import { tic } from 'myst-cli-utils';
 import { TexParser } from 'tex-to-myst';
 import { VFile } from 'vfile';
-import type { GenericParent } from 'myst-common';
 import { RuleId, toText } from 'myst-common';
 import { validatePageFrontmatter } from 'myst-frontmatter';
 import { SourceFileKind } from 'myst-spec-ext';
@@ -18,8 +17,6 @@ import { addWarningForFile } from '../utils/addWarningForFile.js';
 import { loadCitations } from './citations.js';
 import { parseMyst } from './myst.js';
 import { processNotebook } from './notebook.js';
-import { includeDirectiveTransform } from 'myst-transforms';
-import { makeFileLoader } from '../transforms/include.js';
 import { selectors } from '../store/index.js';
 
 function checkCache(cache: ISessionWithCache, content: string, file: string) {
@@ -116,13 +113,6 @@ export async function loadFile(
         const { sha256, useCache } = checkCache(cache, content, file);
         if (useCache) break;
         const tex = new TexParser(content, vfile);
-        await includeDirectiveTransform(tex.ast as GenericParent, vfile, {
-          loadFile: makeFileLoader(session, vfile, file),
-          parseContent: (filename, input) => {
-            const subTex = new TexParser(input, vfile);
-            return subTex.ast.children ?? [];
-          },
-        });
         const frontmatter = validatePageFrontmatter(
           {
             title: toText(tex.data.frontmatter.title as any),

packages/myst-cli/src/transforms/include.ts

@@ -7,24 +7,64 @@ import type { VFile } from 'vfile';
 import { parseMyst } from '../process/myst.js';
 import type { ISession } from '../session/types.js';
 import { watch } from '../store/reducers.js';
+import { TexParser } from 'tex-to-myst';
 
-export const makeFileLoader =
-  (session: ISession, vfile: VFile, baseFile: string) => (filename: string) => {
-    const dir = path.dirname(baseFile);
-    const fullFile = path.join(dir, filename);
+/**
+ * Return resolveFile function
+ *
+ * If `sourceFile` is format .tex, `relativeFile` will be resolved relative to the
+ * original baseFile; otherwise, it will be resolved relative to `sourceFile`.
+ *
+ * The returned function will resolve the file as described above, and return it if
+ * it exists or log an error and return undefined otherwise.
+ */
+export const makeFileResolver =
+  (baseFile: string) => (relativeFile: string, sourceFile: string, vfile: VFile) => {
+    const base = sourceFile.toLowerCase().endsWith('.tex') ? baseFile : sourceFile;
+    const fullFile = path.resolve(path.dirname(base), relativeFile);
     if (!fs.existsSync(fullFile)) {
-      fileError(vfile, `Include Directive: Could not find "${fullFile}" in "${baseFile}"`, {
-        ruleId: RuleId.includeContentLoads,
-      });
+      fileError(
+        vfile,
+        `Include Directive: Could not find "${relativeFile}" relative to "${base}"`,
+        {
+          ruleId: RuleId.includeContentLoads,
+        },
+      );
       return;
     }
-    session.store.dispatch(
-      watch.actions.addLocalDependency({
-        path: baseFile,
-        dependency: fullFile,
-      }),
-    );
-    return fs.readFileSync(fullFile).toString();
+    return fullFile;
+  };
+
+/**
+ * Return loadFile function
+ *
+ * Loaded file is added to original baseFile's dependencies.
+ */
+export const makeFileLoader = (session: ISession, baseFile: string) => (fullFile: string) => {
+  session.store.dispatch(
+    watch.actions.addLocalDependency({
+      path: baseFile,
+      dependency: fullFile,
+    }),
+  );
+  return fs.readFileSync(fullFile).toString();
+};
+
+/**
+ * Return paresContent function
+ *
+ * Handles html and tex files separately; all other files are treated as MyST md.
+ */
+export const makeContentParser =
+  (session: ISession) => (filename: string, content: string, vfile: VFile) => {
+    if (filename.toLowerCase().endsWith('.html')) {
+      return [{ type: 'html', value: content }];
+    }
+    if (filename.toLowerCase().endsWith('.tex')) {
+      const subTex = new TexParser(content, vfile);
+      return subTex.ast.children ?? [];
+    }
+    return parseMyst(session, content, filename).children;
   };
 
 export async function includeFilesTransform(
@@ -33,12 +73,13 @@ export async function includeFilesTransform(
   tree: GenericParent,
   vfile: VFile,
 ) {
-  const parseContent = (filename: string, content: string) => {
-    if (filename.toLowerCase().endsWith('.html')) {
-      return [{ type: 'html', value: content }];
-    }
-    return parseMyst(session, content, filename).children;
-  };
-  const loadFile = makeFileLoader(session, vfile, baseFile);
-  await includeDirectiveTransform(tree, vfile, { loadFile, parseContent });
+  const parseContent = makeContentParser(session);
+  const loadFile = makeFileLoader(session, baseFile);
+  const resolveFile = makeFileResolver(baseFile);
+  await includeDirectiveTransform(tree, vfile, {
+    resolveFile,
+    loadFile,
+    parseContent,
+    sourceFile: baseFile,
+  });
 }

packages/myst-to-typst/src/index.ts

@@ -304,10 +304,10 @@ const handlers: Record<string, Handler> = {
     state.write(`)`);
   },
   embed(node, state) {
-    state.renderChildren(node);
+    state.renderChildren(node, 2);
   },
   include(node, state) {
-    state.renderChildren(node);
+    state.renderChildren(node, 2);
   },
   footnoteReference(node, state) {
     if (!node.identifier) return;

packages/myst-transforms/src/include.ts

@@ -1,5 +1,5 @@
 import type { GenericNode, GenericParent } from 'myst-common';
-import { fileWarn, RuleId } from 'myst-common';
+import { fileError, fileWarn, RuleId } from 'myst-common';
 import type { Code, Container, Include } from 'myst-spec-ext';
 import { selectAll } from 'unist-util-select';
 import type { Caption } from 'myst-spec';
@@ -8,7 +8,14 @@ import type { VFile } from 'vfile';
 
 export type Options = {
   loadFile: (filename: string) => Promise<string | undefined> | string | undefined;
-  parseContent: (filename: string, content: string) => Promise<GenericNode[]> | GenericNode[];
+  resolveFile: (includeFile: string, sourceFile: string, vfile: VFile) => string | undefined;
+  parseContent: (
+    filename: string,
+    content: string,
+    vfile: VFile,
+  ) => Promise<GenericNode[]> | GenericNode[];
+  sourceFile: string;
+  stack?: string[];
 };
 
 /**
@@ -20,11 +27,22 @@ export type Options = {
 export async function includeDirectiveTransform(tree: GenericParent, vfile: VFile, opts: Options) {
   const includeNodes = selectAll('include', tree) as Include[];
   if (includeNodes.length === 0) return;
+  if (!opts?.stack) opts.stack = [opts.sourceFile];
   await Promise.all(
     includeNodes.map(async (node) => {
       // If the transform has already run, don't run it again!
       if (node.children && node.children.length > 0) return;
-      const rawContent = await opts.loadFile(node.file);
+      const fullFile = opts.resolveFile(node.file, opts.sourceFile, vfile);
+      if (!fullFile) return;
+      // If we encounter the same include file twice in a single stack, return
+      if (opts.stack?.includes(fullFile)) {
+        fileError(vfile, `Include Directive: "${fullFile}" depends on itself`, {
+          ruleId: RuleId.includeContentLoads,
+          note: [...opts.stack, fullFile].join(' > '),
+        });
+        return;
+      }
+      const rawContent = await opts.loadFile(fullFile);
       if (rawContent == null) return;
       const { content, startingLineNumber } = filterIncludedContent(vfile, node.filter, rawContent);
       let children: GenericNode[];
@@ -78,11 +96,16 @@ export async function includeDirectiveTransform(tree: GenericParent, vfile: VFil
           children = [container];
         }
       } else {
-        children = await opts.parseContent(node.file, content);
+        children = await opts.parseContent(fullFile, content, vfile);
       }
       node.children = children as any;
+      if (!node.children?.length) return;
       // Recurse!
-      // await includeDirectiveTransform(node as GenericParent, vfile, opts);
+      await includeDirectiveTransform(node as GenericParent, vfile, {
+        ...opts,
+        stack: [...(opts.stack ?? []), fullFile],
+        sourceFile: fullFile,
+      });
     }),
   );
 }

packages/mystmd/tests/exports.yml

@@ -95,3 +95,15 @@ cases:
     outputs:
       - path: acronym/_build/out.tex
         content: outputs/acronym.tex
+  - title: Recursive includes resolve
+    cwd: include-recursive
+    command: myst build --typst
+    outputs:
+      - path: include-recursive/_build/out.typ
+        content: outputs/include-recursive.typ
+  - title: Circular includes resolve with errors
+    cwd: include-circular
+    command: myst build --typst
+    outputs:
+      - path: include-circular/_build/out.typ
+        content: outputs/include-circular.typ

packages/mystmd/tests/include-circular/dir/three.md

@@ -0,0 +1,6 @@
+# Page three
+
+This page includes content from page one:
+
+```{include} ../one.md
+```

packages/mystmd/tests/include-circular/dir/two.md

@@ -0,0 +1,6 @@
+# Page two
+
+This page includes content from page three:
+
+```{include} ./three.md
+```

packages/mystmd/tests/include-circular/four.md

@@ -0,0 +1,6 @@
+# Page four
+
+This page includes content from itself:
+
+```{include} ./four.md
+```

packages/mystmd/tests/include-circular/myst.yml

@@ -0,0 +1,14 @@
+# See docs at: https://mystmd.org/guide/frontmatter
+version: 1
+project:
+  id: c11009ec-caf7-445c-b498-403bfe2aa61f
+  keywords: []
+  authors: []
+  github: https://github.com/executablebooks/mystjs
+site:
+  template: book-theme
+  nav: []
+  actions:
+    - title: Learn More
+      url: https://mystmd.org/guide
+  domains: []

packages/mystmd/tests/include-circular/one.md

@@ -0,0 +1,21 @@
+---
+export:
+  - _build/out.typ
+---
+
+# Page one
+
+This page includes content from page two:
+
+```{include} ./dir/two.md
+```
+
+and content from page three:
+
+```{include} ./dir/three.md
+```
+
+and page four:
+
+```{include} ./four.md
+```

packages/mystmd/tests/include-recursive/myst.yml

@@ -0,0 +1,14 @@
+# See docs at: https://mystmd.org/guide/frontmatter
+version: 1
+project:
+  id: c11009ec-caf7-445c-b498-403bfe2aa61f
+  keywords: []
+  authors: []
+  github: https://github.com/executablebooks/mystjs
+site:
+  template: book-theme
+  nav: []
+  actions:
+    - title: Learn More
+      url: https://mystmd.org/guide
+  domains: []

packages/mystmd/tests/include-recursive/one.md

@@ -0,0 +1,16 @@
+---
+export:
+  - _build/out.typ
+---
+
+# Page one
+
+This page includes content from page two:
+
+```{include} ./two.md
+```
+
+and content from page three:
+
+```{include} ./three.md
+```

packages/mystmd/tests/include-recursive/three.md

@@ -0,0 +1,3 @@
+# Page three
+
+This page does not include any other pages.

packages/mystmd/tests/include-recursive/two.md

@@ -0,0 +1,6 @@
+# Page two
+
+This page includes content from page three:
+
+```{include} ./three.md
+```