From 0e6054472249b262cf3f0ca4edbf3dc9d49860dd Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Thu, 16 May 2024 00:20:22 -0600
Subject: [PATCH 01/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor=20config=20loadin?=
 =?UTF-8?q?g=20to=20separate=20validation=20from=20saving?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .changeset/stupid-experts-pull.md |   5 +
 packages/myst-cli/src/config.ts   | 160 ++++++++++++++++++------------
 2 files changed, 101 insertions(+), 64 deletions(-)
 create mode 100644 .changeset/stupid-experts-pull.md

diff --git a/.changeset/stupid-experts-pull.md b/.changeset/stupid-experts-pull.md
new file mode 100644
index 000000000..ab9b6ec94
--- /dev/null
+++ b/.changeset/stupid-experts-pull.md
@@ -0,0 +1,5 @@
+---
+'myst-cli': patch
+---
+
+Refactor config loading to separate validation from saving
diff --git a/packages/myst-cli/src/config.ts b/packages/myst-cli/src/config.ts
index 9a48b55eb..f5d264a2c 100644
--- a/packages/myst-cli/src/config.ts
+++ b/packages/myst-cli/src/config.ts
@@ -41,18 +41,9 @@ export function configFromPath(session: ISession, path: string) {
 }
 
 /**
- * Load site/project config from local path to redux store
- *
- * Errors if config file does not exist or if config file exists but is invalid.
+ * Load config yaml file and throw error if it fails
  */
-export function loadConfig(session: ISession, path: string) {
-  const file = configFromPath(session, path);
-  if (!file) {
-    session.log.debug(`No config loaded from path: ${path}`);
-    return;
-  }
-  const vfile = new VFile();
-  vfile.path = file;
+function loadConfigYaml(file: string) {
   if (!fs.existsSync(file)) throw Error(`Cannot find config file: ${file}`);
   let rawConf: Record<string, any>;
   try {
@@ -61,22 +52,38 @@ export function loadConfig(session: ISession, path: string) {
     const suffix = (err as Error).message ? `\n\n${(err as Error).message}` : '';
     throw Error(`Unable to read config file ${file} as YAML${suffix}`);
   }
-  const existingConf = selectors.selectLocalRawConfig(session.store.getState(), path);
-  if (existingConf && JSON.stringify(rawConf) === JSON.stringify(existingConf.raw)) {
-    return existingConf.validated;
-  }
-  const opts: ValidationOptions = {
-    file,
-    property: 'config',
+  return rawConf;
+}
+
+/**
+ * Helper function to generate basic validation options
+ */
+function configValidationOpts(vfile: VFile, property: string, ruleId: RuleId): ValidationOptions {
+  return {
+    file: vfile.path,
+    property,
     messages: {},
     errorLogFn: (message: string) => {
-      fileError(vfile, message, { ruleId: RuleId.validConfigStructure });
+      fileError(vfile, message, { ruleId });
     },
     warningLogFn: (message: string) => {
-      fileWarn(vfile, message, { ruleId: RuleId.validConfigStructure });
+      fileWarn(vfile, message, { ruleId });
     },
   };
-  const conf = validateObject(yaml.load(fs.readFileSync(file, 'utf-8')), opts);
+}
+
+/**
+ * Load and validate a file as yaml config file
+ *
+ * Returns validated site and project configs.
+ *
+ * Throws errors config file is malformed or invalid.
+ */
+function getValidatedConfigsFromFile(session: ISession, file: string) {
+  const vfile = new VFile();
+  vfile.path = file;
+  const opts = configValidationOpts(vfile, 'config', RuleId.validConfigStructure);
+  const conf = validateObject(loadConfigYaml(file), opts);
   if (conf) {
     const filteredConf = validateKeys(
       conf,
@@ -120,24 +127,49 @@ export function loadConfig(session: ISession, path: string) {
     const { logoText, ...rest } = conf.site;
     conf.site = { logo_text: logoText, ...rest };
   }
-  session.store.dispatch(
-    config.actions.receiveRawConfig({ path, file, raw: rawConf, validated: conf }),
-  );
-  const { site, project } = conf ?? {};
-  if (site) {
-    validateSiteConfigAndSave(session, path, vfile, site);
+  const { site: rawSite, project: rawProject } = conf ?? {};
+  const path = dirname(file);
+  let site: SiteConfig | undefined;
+  let project: ProjectConfig | undefined;
+  if (rawSite) {
+    site = validateSiteConfigAndThrow(session, path, vfile, rawSite);
     session.log.debug(`Loaded site config from ${file}`);
   } else {
     session.log.debug(`No site config in ${file}`);
   }
-  if (project) {
-    validateProjectConfigAndSave(session, path, vfile, project);
+  if (rawProject) {
+    project = validateProjectConfigAndThrow(session, path, vfile, rawProject);
     session.log.debug(`Loaded project config from ${file}`);
   } else {
     session.log.debug(`No project config defined in ${file}`);
   }
   logMessagesFromVFile(session, vfile);
-  return conf;
+  return { site, project };
+}
+
+/**
+ * Load site/project config from local path to redux store
+ *
+ * Errors if config file does not exist or if config file exists but is invalid.
+ */
+export function loadConfig(session: ISession, path: string) {
+  const file = configFromPath(session, path);
+  if (!file) {
+    session.log.debug(`No config loaded from path: ${path}`);
+    return;
+  }
+  const rawConf = loadConfigYaml(file);
+  const existingConf = selectors.selectLocalRawConfig(session.store.getState(), path);
+  if (existingConf && JSON.stringify(rawConf) === JSON.stringify(existingConf.raw)) {
+    return existingConf.validated;
+  }
+  const { site, project } = getValidatedConfigsFromFile(session, file);
+  session.store.dispatch(
+    config.actions.receiveRawConfig({ path, file, raw: rawConf, validated: { site, project } }),
+  );
+  if (site) saveSiteConfig(session, path, site);
+  if (project) saveProjectConfig(session, path, project);
+  return { site, project };
 }
 
 export function resolveToAbsolute(
@@ -241,56 +273,48 @@ function resolveProjectConfigPaths(
   return { ...projectConfig, ...resolvedFields };
 }
 
-function validateSiteConfigAndSave(
+function validateSiteConfigAndThrow(
   session: ISession,
   path: string,
   vfile: VFile,
-  rawSiteConfig: Record<string, any>,
+  rawSite: Record<string, any>,
 ) {
-  let siteConfig = validateSiteConfig(rawSiteConfig, {
-    file: vfile.path,
-    property: 'site',
-    messages: {},
-    errorLogFn: (message: string) => {
-      fileError(vfile, message, { ruleId: RuleId.validSiteConfig });
-    },
-    warningLogFn: (message: string) => {
-      fileWarn(vfile, message, { ruleId: RuleId.validSiteConfig });
-    },
-  });
+  const site = validateSiteConfig(
+    rawSite,
+    configValidationOpts(vfile, 'config.site', RuleId.validSiteConfig),
+  );
   logMessagesFromVFile(session, vfile);
-  if (!siteConfig) {
+  if (!site) {
     const errorSuffix = vfile.path ? ` in ${vfile.path}` : '';
     throw Error(`Please address invalid site config${errorSuffix}`);
   }
-  siteConfig = resolveSiteConfigPaths(session, path, siteConfig, resolveToAbsolute);
-  session.store.dispatch(config.actions.receiveSiteConfig({ path, ...siteConfig }));
+  return resolveSiteConfigPaths(session, path, site, resolveToAbsolute);
 }
 
-function validateProjectConfigAndSave(
+function saveSiteConfig(session: ISession, path: string, site: SiteConfig) {
+  session.store.dispatch(config.actions.receiveSiteConfig({ path, ...site }));
+}
+
+function validateProjectConfigAndThrow(
   session: ISession,
   path: string,
   vfile: VFile,
-  rawProjectConfig: Record<string, any>,
+  rawProject: Record<string, any>,
 ) {
-  let projectConfig = validateProjectConfig(rawProjectConfig, {
-    file: vfile.path,
-    property: 'project',
-    messages: {},
-    errorLogFn: (message: string) => {
-      fileError(vfile, message, { ruleId: RuleId.validProjectConfig });
-    },
-    warningLogFn: (message: string) => {
-      fileWarn(vfile, message, { ruleId: RuleId.validProjectConfig });
-    },
-  });
+  const project = validateProjectConfig(
+    rawProject,
+    configValidationOpts(vfile, 'config.project', RuleId.validProjectConfig),
+  );
   logMessagesFromVFile(session, vfile);
-  if (!projectConfig) {
+  if (!project) {
     const errorSuffix = vfile.path ? ` in ${vfile.path}` : '';
     throw Error(`Please address invalid project config${errorSuffix}`);
   }
-  projectConfig = resolveProjectConfigPaths(session, path, projectConfig, resolveToAbsolute);
-  session.store.dispatch(config.actions.receiveProjectConfig({ path, ...projectConfig }));
+  return resolveProjectConfigPaths(session, path, project, resolveToAbsolute);
+}
+
+function saveProjectConfig(session: ISession, path: string, project: ProjectConfig) {
+  session.store.dispatch(config.actions.receiveProjectConfig({ path, ...project }));
 }
 
 /**
@@ -317,13 +341,21 @@ export function writeConfigs(
   // Get site config to save
   const vfile = new VFile();
   vfile.path = file;
-  if (siteConfig) validateSiteConfigAndSave(session, path, vfile, siteConfig);
+  if (siteConfig) {
+    saveSiteConfig(session, path, validateSiteConfigAndThrow(session, path, vfile, siteConfig));
+  }
   siteConfig = selectors.selectLocalSiteConfig(session.store.getState(), path);
   if (siteConfig) {
     siteConfig = resolveSiteConfigPaths(session, path, siteConfig, resolveToRelative);
   }
   // Get project config to save
-  if (projectConfig) validateProjectConfigAndSave(session, path, vfile, projectConfig);
+  if (projectConfig) {
+    saveProjectConfig(
+      session,
+      path,
+      validateProjectConfigAndThrow(session, path, vfile, projectConfig),
+    );
+  }
   projectConfig = selectors.selectLocalProjectConfig(session.store.getState(), path);
   if (projectConfig) {
     projectConfig = prepareToWrite(projectConfig);

From b2abb890634f0c073a400cb6e6891f6fc2468fcd Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Thu, 16 May 2024 01:32:43 -0600
Subject: [PATCH 02/11] =?UTF-8?q?=F0=9F=8F=84=E2=80=8D=E2=99=80=EF=B8=8F?=
 =?UTF-8?q?=20=20Add=20extend=20key=20to=20top-level=20config?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .changeset/many-worms-know.md     |  6 +++++
 packages/myst-cli/src/config.ts   | 42 +++++++++++++++++++++----------
 packages/myst-config/src/index.ts |  1 +
 3 files changed, 36 insertions(+), 13 deletions(-)
 create mode 100644 .changeset/many-worms-know.md

diff --git a/.changeset/many-worms-know.md b/.changeset/many-worms-know.md
new file mode 100644
index 000000000..3ce655570
--- /dev/null
+++ b/.changeset/many-worms-know.md
@@ -0,0 +1,6 @@
+---
+'myst-config': patch
+'myst-cli': patch
+---
+
+Add extend key to top-level config
diff --git a/packages/myst-cli/src/config.ts b/packages/myst-cli/src/config.ts
index f5d264a2c..3f6e91af1 100644
--- a/packages/myst-cli/src/config.ts
+++ b/packages/myst-cli/src/config.ts
@@ -6,7 +6,13 @@ import { fileError, fileWarn, RuleId } from 'myst-common';
 import type { Config, ProjectConfig, SiteConfig, SiteProject } from 'myst-config';
 import { validateProjectConfig, validateSiteConfig } from 'myst-config';
 import type { ValidationOptions } from 'simple-validators';
-import { incrementOptions, validateKeys, validateObject, validationError } from 'simple-validators';
+import {
+  incrementOptions,
+  validateObjectKeys,
+  validationError,
+  validateList,
+  validateString,
+} from 'simple-validators';
 import { VFile } from 'vfile';
 import { prepareToWrite } from './frontmatter.js';
 import type { ISession } from './session/types.js';
@@ -83,19 +89,20 @@ function getValidatedConfigsFromFile(session: ISession, file: string) {
   const vfile = new VFile();
   vfile.path = file;
   const opts = configValidationOpts(vfile, 'config', RuleId.validConfigStructure);
-  const conf = validateObject(loadConfigYaml(file), opts);
-  if (conf) {
-    const filteredConf = validateKeys(
-      conf,
-      { required: ['version'], optional: ['site', 'project'] },
-      opts,
+  const conf = validateObjectKeys(
+    loadConfigYaml(file),
+    {
+      required: ['version'],
+      optional: ['site', 'project', 'extend'],
+      alias: { extends: 'extend' },
+    },
+    opts,
+  );
+  if (conf && conf.version !== VERSION) {
+    validationError(
+      `"${conf.version}" does not match ${VERSION}`,
+      incrementOptions('version', opts),
     );
-    if (filteredConf && filteredConf.version !== VERSION) {
-      validationError(
-        `"${filteredConf.version}" does not match ${VERSION}`,
-        incrementOptions('version', opts),
-      );
-    }
   }
   logMessagesFromVFile(session, vfile);
   if (!conf || opts.messages.errors) {
@@ -127,6 +134,15 @@ function getValidatedConfigsFromFile(session: ISession, file: string) {
     const { logoText, ...rest } = conf.site;
     conf.site = { logo_text: logoText, ...rest };
   }
+  const extend = validateList(
+    conf.extend,
+    { coerce: true, ...incrementOptions('extend', opts) },
+    (item, index) => {
+      const relativeFile = validateString(item, incrementOptions(`extend.${index}`, opts));
+      if (!relativeFile) return relativeFile;
+      return resolveToAbsolute(session, dirname(file), relativeFile);
+    },
+  );
   const { site: rawSite, project: rawProject } = conf ?? {};
   const path = dirname(file);
   let site: SiteConfig | undefined;
diff --git a/packages/myst-config/src/index.ts b/packages/myst-config/src/index.ts
index 2f783637c..6c3e5f84a 100644
--- a/packages/myst-config/src/index.ts
+++ b/packages/myst-config/src/index.ts
@@ -6,6 +6,7 @@ export * from './site/index.js';
 
 export type Config = {
   version: 1;
+  extend?: string[];
   project?: ProjectConfig;
   site?: SiteConfig;
 };

From 2a4084acbb227e0b7e0de713bb1e24b00748b769 Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Thu, 16 May 2024 07:53:08 -0600
Subject: [PATCH 03/11] =?UTF-8?q?=E2=9B=BD=EF=B8=8F=20Load=20and=20fill=20?=
 =?UTF-8?q?frontmatter=20from=20extend=20config=20key?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .changeset/large-peas-melt.md                 |  6 ++
 packages/myst-cli/src/config.ts               | 59 ++++++++++++++-----
 .../src/utils/fillPageFrontmatter.ts          | 50 +++++++++-------
 3 files changed, 78 insertions(+), 37 deletions(-)
 create mode 100644 .changeset/large-peas-melt.md

diff --git a/.changeset/large-peas-melt.md b/.changeset/large-peas-melt.md
new file mode 100644
index 000000000..dbd160a02
--- /dev/null
+++ b/.changeset/large-peas-melt.md
@@ -0,0 +1,6 @@
+---
+'myst-frontmatter': patch
+'myst-cli': patch
+---
+
+Load and fill frontmatter from extend config key
diff --git a/packages/myst-cli/src/config.ts b/packages/myst-cli/src/config.ts
index 3f6e91af1..8f61d1777 100644
--- a/packages/myst-cli/src/config.ts
+++ b/packages/myst-cli/src/config.ts
@@ -5,6 +5,7 @@ import { writeFileToFolder } from 'myst-cli-utils';
 import { fileError, fileWarn, RuleId } from 'myst-common';
 import type { Config, ProjectConfig, SiteConfig, SiteProject } from 'myst-config';
 import { validateProjectConfig, validateSiteConfig } from 'myst-config';
+import { fillProjectFrontmatter } from 'myst-frontmatter';
 import type { ValidationOptions } from 'simple-validators';
 import {
   incrementOptions,
@@ -12,6 +13,7 @@ import {
   validationError,
   validateList,
   validateString,
+  fillMissingKeys,
 } from 'simple-validators';
 import { VFile } from 'vfile';
 import { prepareToWrite } from './frontmatter.js';
@@ -78,6 +80,13 @@ function configValidationOpts(vfile: VFile, property: string, ruleId: RuleId): V
   };
 }
 
+/**
+ * Function to add filler keys to base if the keys are not defined in base
+ */
+function fillSiteConfig(base: SiteConfig, filler: SiteConfig) {
+  return fillMissingKeys(base, filler, Object.keys(filler));
+}
+
 /**
  * Load and validate a file as yaml config file
  *
@@ -134,27 +143,47 @@ function getValidatedConfigsFromFile(session: ISession, file: string) {
     const { logoText, ...rest } = conf.site;
     conf.site = { logo_text: logoText, ...rest };
   }
-  const extend = validateList(
-    conf.extend,
-    { coerce: true, ...incrementOptions('extend', opts) },
-    (item, index) => {
-      const relativeFile = validateString(item, incrementOptions(`extend.${index}`, opts));
-      if (!relativeFile) return relativeFile;
-      return resolveToAbsolute(session, dirname(file), relativeFile);
-    },
-  );
-  const { site: rawSite, project: rawProject } = conf ?? {};
-  const path = dirname(file);
   let site: SiteConfig | undefined;
   let project: ProjectConfig | undefined;
+  const projectOpts = configValidationOpts(vfile, 'config.project', RuleId.validProjectConfig);
+  if (conf.extend) {
+    const extend = validateList(
+      conf.extend,
+      { coerce: true, ...incrementOptions('extend', opts) },
+      (item, index) => {
+        const relativeFile = validateString(item, incrementOptions(`extend.${index}`, opts));
+        if (!relativeFile) return relativeFile;
+        return resolveToAbsolute(session, dirname(file), relativeFile);
+      },
+    );
+    extend?.forEach((extFile) => {
+      const { site: extSite, project: extProject } = getValidatedConfigsFromFile(session, extFile);
+      if (extSite) {
+        site = site ? fillSiteConfig(extSite, site) : extSite;
+      }
+      if (extProject) {
+        project = project ? fillProjectFrontmatter(extProject, project, projectOpts) : extProject;
+      }
+    });
+  }
+  const { site: rawSite, project: rawProject } = conf ?? {};
+  const path = dirname(file);
   if (rawSite) {
-    site = validateSiteConfigAndThrow(session, path, vfile, rawSite);
+    site = fillSiteConfig(validateSiteConfigAndThrow(session, path, vfile, rawSite), site ?? {});
+  }
+  if (site) {
     session.log.debug(`Loaded site config from ${file}`);
   } else {
     session.log.debug(`No site config in ${file}`);
   }
   if (rawProject) {
-    project = validateProjectConfigAndThrow(session, path, vfile, rawProject);
+    project = fillProjectFrontmatter(
+      validateProjectConfigAndThrow(session, path, vfile, rawProject),
+      project ?? {},
+      projectOpts,
+    );
+  }
+  if (project) {
     session.log.debug(`Loaded project config from ${file}`);
   } else {
     session.log.debug(`No project config defined in ${file}`);
@@ -294,7 +323,7 @@ function validateSiteConfigAndThrow(
   path: string,
   vfile: VFile,
   rawSite: Record<string, any>,
-) {
+): SiteConfig {
   const site = validateSiteConfig(
     rawSite,
     configValidationOpts(vfile, 'config.site', RuleId.validSiteConfig),
@@ -316,7 +345,7 @@ function validateProjectConfigAndThrow(
   path: string,
   vfile: VFile,
   rawProject: Record<string, any>,
-) {
+): ProjectConfig {
   const project = validateProjectConfig(
     rawProject,
     configValidationOpts(vfile, 'config.project', RuleId.validProjectConfig),
diff --git a/packages/myst-frontmatter/src/utils/fillPageFrontmatter.ts b/packages/myst-frontmatter/src/utils/fillPageFrontmatter.ts
index af3550c09..3bd6f224a 100644
--- a/packages/myst-frontmatter/src/utils/fillPageFrontmatter.ts
+++ b/packages/myst-frontmatter/src/utils/fillPageFrontmatter.ts
@@ -19,39 +19,48 @@ export function fillPageFrontmatter(
   pageFrontmatter: PageFrontmatter,
   projectFrontmatter: ProjectFrontmatter,
   opts: ValidationOptions,
+): PageFrontmatter {
+  return fillProjectFrontmatter(pageFrontmatter, projectFrontmatter, opts, USE_PROJECT_FALLBACK);
+}
+
+export function fillProjectFrontmatter(
+  base: ProjectFrontmatter,
+  filler: ProjectFrontmatter,
+  opts: ValidationOptions,
+  keys?: string[],
 ) {
-  const frontmatter = fillMissingKeys(pageFrontmatter, projectFrontmatter, USE_PROJECT_FALLBACK);
+  const frontmatter = fillMissingKeys(base, filler, keys ?? Object.keys(filler));
 
-  if (pageFrontmatter.numbering || projectFrontmatter.numbering) {
-    frontmatter.numbering = fillNumbering(pageFrontmatter.numbering, projectFrontmatter.numbering);
+  if (filler.numbering || base.numbering) {
+    frontmatter.numbering = fillNumbering(base.numbering, filler.numbering);
   }
 
   // Combine all math macros defined on page and project
-  if (projectFrontmatter.math || pageFrontmatter.math) {
-    frontmatter.math = { ...(projectFrontmatter.math ?? {}), ...(pageFrontmatter.math ?? {}) };
+  if (filler.math || base.math) {
+    frontmatter.math = { ...(filler.math ?? {}), ...(base.math ?? {}) };
   }
 
   // Combine all abbreviation defined on page and project
-  if (projectFrontmatter.abbreviations || pageFrontmatter.abbreviations) {
+  if (filler.abbreviations || base.abbreviations) {
     frontmatter.abbreviations = {
-      ...(projectFrontmatter.abbreviations ?? {}),
-      ...(pageFrontmatter.abbreviations ?? {}),
+      ...(filler.abbreviations ?? {}),
+      ...(base.abbreviations ?? {}),
     };
   }
 
   // Combine all options defined on page and project
-  if (projectFrontmatter.options || pageFrontmatter.options) {
+  if (filler.options || base.options) {
     frontmatter.options = {
-      ...(projectFrontmatter.options ?? {}),
-      ...(pageFrontmatter.options ?? {}),
+      ...(filler.options ?? {}),
+      ...(base.options ?? {}),
     };
   }
 
   // Combine all settings defined on page and project
-  if (projectFrontmatter.settings || pageFrontmatter.settings) {
+  if (filler.settings || base.settings) {
     frontmatter.settings = {
-      ...(projectFrontmatter.settings ?? {}),
-      ...(pageFrontmatter.settings ?? {}),
+      ...(filler.settings ?? {}),
+      ...(base.settings ?? {}),
     };
   }
 
@@ -83,10 +92,10 @@ export function fillPageFrontmatter(
   if (frontmatter.authors?.length || contributorIds.size) {
     // Gather all people from page/project authors/contributors
     const people = [
-      ...(pageFrontmatter.authors ?? []),
-      ...(projectFrontmatter.authors ?? []),
-      ...(pageFrontmatter.contributors ?? []),
-      ...(projectFrontmatter.contributors ?? []),
+      ...(base.authors ?? []),
+      ...(filler.authors ?? []),
+      ...(base.contributors ?? []),
+      ...(filler.contributors ?? []),
     ];
     const peopleLookup: Record<string, Contributor> = {};
     people.forEach((auth) => {
@@ -126,10 +135,7 @@ export function fillPageFrontmatter(
   });
 
   if (affiliationIds.size) {
-    const affiliations = [
-      ...(pageFrontmatter.affiliations ?? []),
-      ...(projectFrontmatter.affiliations ?? []),
-    ];
+    const affiliations = [...(base.affiliations ?? []), ...(filler.affiliations ?? [])];
     const affiliationLookup: Record<string, Affiliation> = {};
     affiliations.forEach((aff) => {
       if (!aff.id || isStashPlaceholder(aff)) return;

From 9eea2562efb66946a0ff3f8c0067d35aec91de07 Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Thu, 16 May 2024 22:23:09 -0600
Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=A7=AA=20Tidy=20myst=20version=20in?=
 =?UTF-8?q?=20existing=20e2e=20tests?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../tests/outputs/basic-site-config.json      | 20 ++++++++++++++++++-
 .../tests/outputs/site-xrefs-config.json      |  1 -
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/packages/mystmd/tests/outputs/basic-site-config.json b/packages/mystmd/tests/outputs/basic-site-config.json
index 146e36ae7..e306b906c 100644
--- a/packages/mystmd/tests/outputs/basic-site-config.json
+++ b/packages/mystmd/tests/outputs/basic-site-config.json
@@ -1 +1,19 @@
-{"options":{},"myst":"1.2.3","nav":[],"actions":[{"title":"Learn More","url":"https://mystmd.org/guide","internal":false,"static":false}],"projects":[{"github":"https://github.com/executablebooks/mystjs","keywords":[],"id":"22c218e1-66c6-428f-9df9-a7f2c6a3bd76","exports":[],"bibliography":[],"title":"Basic Test","index":"index","pages":[]}]}
\ No newline at end of file
+{
+  "options": {},
+  "nav": [],
+  "actions": [
+    { "title": "Learn More", "url": "https://mystmd.org/guide", "internal": false, "static": false }
+  ],
+  "projects": [
+    {
+      "github": "https://github.com/executablebooks/mystjs",
+      "keywords": [],
+      "id": "22c218e1-66c6-428f-9df9-a7f2c6a3bd76",
+      "exports": [],
+      "bibliography": [],
+      "title": "Basic Test",
+      "index": "index",
+      "pages": []
+    }
+  ]
+}
diff --git a/packages/mystmd/tests/outputs/site-xrefs-config.json b/packages/mystmd/tests/outputs/site-xrefs-config.json
index 32b432806..5f05c6509 100644
--- a/packages/mystmd/tests/outputs/site-xrefs-config.json
+++ b/packages/mystmd/tests/outputs/site-xrefs-config.json
@@ -1,6 +1,5 @@
 {
   "options": {},
-  "myst": "1.2.3",
   "nav": [],
   "actions": [
     { "title": "Learn More", "url": "https://mystmd.org/guide", "internal": false, "static": false }

From 60790a8801bf2d22c16b3838fe0cceba1d8c3777 Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Thu, 16 May 2024 22:25:37 -0600
Subject: [PATCH 05/11] =?UTF-8?q?=F0=9F=94=84=20Support=20circular=20deps?=
 =?UTF-8?q?=20and=20live=20reloading=20for=20extending=20config?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .changeset/green-brooms-applaud.md        |  5 ++
 packages/myst-cli/src/build/site/watch.ts | 12 +++--
 packages/myst-cli/src/config.ts           | 59 +++++++++++++++++------
 packages/myst-cli/src/project/load.ts     |  1 +
 packages/myst-cli/src/store/reducers.ts   | 11 +++--
 packages/myst-cli/src/store/selectors.ts  |  8 ++-
 packages/myst-cli/src/store/types.ts      |  6 +++
 7 files changed, 79 insertions(+), 23 deletions(-)
 create mode 100644 .changeset/green-brooms-applaud.md

diff --git a/.changeset/green-brooms-applaud.md b/.changeset/green-brooms-applaud.md
new file mode 100644
index 000000000..18f4a4ae2
--- /dev/null
+++ b/.changeset/green-brooms-applaud.md
@@ -0,0 +1,5 @@
+---
+'myst-cli': patch
+---
+
+Support circular deps and live reloading for extending config
diff --git a/packages/myst-cli/src/build/site/watch.ts b/packages/myst-cli/src/build/site/watch.ts
index 777b17b69..8aea18f1f 100644
--- a/packages/myst-cli/src/build/site/watch.ts
+++ b/packages/myst-cli/src/build/site/watch.ts
@@ -16,8 +16,10 @@ function watchConfigAndPublic(
   opts: ProcessSiteOptions,
 ) {
   const watchFiles = ['public'];
-  const siteConfigFile = selectors.selectCurrentSiteFile(session.store.getState());
+  const state = session.store.getState();
+  const siteConfigFile = selectors.selectCurrentSiteFile(state);
   if (siteConfigFile) watchFiles.push(siteConfigFile);
+  watchFiles.push(...selectors.selectConfigExtensions(state));
   return chokidar
     .watch(watchFiles, {
       ignoreInitial: true,
@@ -37,6 +39,7 @@ function triggerProjectReload(
   const projectConfigFile = projectPath
     ? selectors.selectLocalConfigFile(state, projectPath)
     : selectors.selectCurrentProjectFile(state);
+  if (selectors.selectConfigExtensions(state).includes(file)) return true;
   if (file === projectConfigFile || basename(file) === '_toc.yml') return true;
   // Reload project if file is added or remvoed
   if (['add', 'unlink'].includes(eventType)) return true;
@@ -64,8 +67,11 @@ async function processorFn(
     !KNOWN_FAST_BUILDS.has(extname(file)) ||
     ['add', 'unlink'].includes(eventType)
   ) {
-    let reloadProject = false;
-    if (file && triggerProjectReload(session, file, eventType, siteProject?.path)) {
+    let reloadProject = opts?.reloadProject ?? false;
+    if (
+      reloadProject ||
+      (file && triggerProjectReload(session, file, eventType, siteProject?.path))
+    ) {
       session.log.info('💥 Triggered full project load and site rebuild');
       reloadProject = true;
     } else {
diff --git a/packages/myst-cli/src/config.ts b/packages/myst-cli/src/config.ts
index 8f61d1777..0614b1002 100644
--- a/packages/myst-cli/src/config.ts
+++ b/packages/myst-cli/src/config.ts
@@ -94,9 +94,16 @@ function fillSiteConfig(base: SiteConfig, filler: SiteConfig) {
  *
  * Throws errors config file is malformed or invalid.
  */
-function getValidatedConfigsFromFile(session: ISession, file: string) {
-  const vfile = new VFile();
-  vfile.path = file;
+function getValidatedConfigsFromFile(
+  session: ISession,
+  file: string,
+  vfile?: VFile,
+  stack?: string[],
+) {
+  if (!vfile) {
+    vfile = new VFile();
+    vfile.path = file;
+  }
   const opts = configValidationOpts(vfile, 'config', RuleId.validConfigStructure);
   const conf = validateObjectKeys(
     loadConfigYaml(file),
@@ -146,8 +153,9 @@ function getValidatedConfigsFromFile(session: ISession, file: string) {
   let site: SiteConfig | undefined;
   let project: ProjectConfig | undefined;
   const projectOpts = configValidationOpts(vfile, 'config.project', RuleId.validProjectConfig);
+  let extend: string[] | undefined;
   if (conf.extend) {
-    const extend = validateList(
+    extend = validateList(
       conf.extend,
       { coerce: true, ...incrementOptions('extend', opts) },
       (item, index) => {
@@ -156,8 +164,22 @@ function getValidatedConfigsFromFile(session: ISession, file: string) {
         return resolveToAbsolute(session, dirname(file), relativeFile);
       },
     );
+    stack = [...(stack ?? []), file];
     extend?.forEach((extFile) => {
-      const { site: extSite, project: extProject } = getValidatedConfigsFromFile(session, extFile);
+      if (stack?.includes(extFile)) {
+        fileError(vfile, 'Circular dependency encountered during "config.extend" resolution', {
+          ruleId: RuleId.validConfigStructure,
+          note: [...stack, extFile].map((f) => resolveToRelative(session, '.', f)).join(' > '),
+        });
+        return;
+      }
+      const { site: extSite, project: extProject } = getValidatedConfigsFromFile(
+        session,
+        extFile,
+        vfile,
+        stack,
+      );
+      session.store.dispatch(config.actions.receiveConfigExtension({ file: extFile }));
       if (extSite) {
         site = site ? fillSiteConfig(extSite, site) : extSite;
       }
@@ -189,7 +211,7 @@ function getValidatedConfigsFromFile(session: ISession, file: string) {
     session.log.debug(`No project config defined in ${file}`);
   }
   logMessagesFromVFile(session, vfile);
-  return { site, project };
+  return { site, project, extend };
 }
 
 /**
@@ -197,24 +219,32 @@ function getValidatedConfigsFromFile(session: ISession, file: string) {
  *
  * Errors if config file does not exist or if config file exists but is invalid.
  */
-export function loadConfig(session: ISession, path: string) {
+export function loadConfig(session: ISession, path: string, opts?: { reloadProject?: boolean }) {
   const file = configFromPath(session, path);
   if (!file) {
     session.log.debug(`No config loaded from path: ${path}`);
     return;
   }
   const rawConf = loadConfigYaml(file);
-  const existingConf = selectors.selectLocalRawConfig(session.store.getState(), path);
-  if (existingConf && JSON.stringify(rawConf) === JSON.stringify(existingConf.raw)) {
-    return existingConf.validated;
+  if (!opts?.reloadProject) {
+    const existingConf = selectors.selectLocalRawConfig(session.store.getState(), path);
+    if (existingConf && JSON.stringify(rawConf) === JSON.stringify(existingConf.raw)) {
+      return existingConf.validated;
+    }
   }
-  const { site, project } = getValidatedConfigsFromFile(session, file);
+  const { site, project, extend } = getValidatedConfigsFromFile(session, file);
+  const validated = { ...rawConf, site, project, extend };
   session.store.dispatch(
-    config.actions.receiveRawConfig({ path, file, raw: rawConf, validated: { site, project } }),
+    config.actions.receiveRawConfig({
+      path,
+      file,
+      raw: rawConf,
+      validated,
+    }),
   );
   if (site) saveSiteConfig(session, path, site);
   if (project) saveProjectConfig(session, path, project);
-  return { site, project };
+  return validated;
 }
 
 export function resolveToAbsolute(
@@ -412,8 +442,7 @@ export function writeConfigs(
     return;
   }
   // Get raw config to override
-  const rawConfig = loadConfig(session, path);
-  const validatedRawConfig = rawConfig?.validated ?? emptyConfig();
+  const validatedRawConfig = loadConfig(session, path) ?? emptyConfig();
   let logContent: string;
   if (siteConfig && projectConfig) {
     logContent = 'site and project configs';
diff --git a/packages/myst-cli/src/project/load.ts b/packages/myst-cli/src/project/load.ts
index 01b6f843c..f04313375 100644
--- a/packages/myst-cli/src/project/load.ts
+++ b/packages/myst-cli/src/project/load.ts
@@ -38,6 +38,7 @@ export async function loadProjectFromDisk(
     const cachedProject = selectors.selectLocalProject(session.store.getState(), path);
     if (cachedProject) return cachedProject;
   }
+  loadConfig(session, path, opts);
   const projectConfig = selectors.selectLocalProjectConfig(session.store.getState(), path);
   const file = join(path, session.configFiles[0]);
   if (!projectConfig && opts?.warnOnNoConfig) {
diff --git a/packages/myst-cli/src/store/reducers.ts b/packages/myst-cli/src/store/reducers.ts
index 9a6936fa9..a5ae886c6 100644
--- a/packages/myst-cli/src/store/reducers.ts
+++ b/packages/myst-cli/src/store/reducers.ts
@@ -3,7 +3,7 @@ import type { PayloadAction } from '@reduxjs/toolkit';
 import { createSlice } from '@reduxjs/toolkit';
 import type { ProjectConfig, SiteConfig } from 'myst-config';
 import { combineReducers } from 'redux';
-import type { BuildWarning, ExternalLinkResult } from './types.js';
+import type { BuildWarning, ExternalLinkResult, ValidatedRawConfig } from './types.js';
 import type { LocalProject } from '../project/types.js';
 
 export const projects = createSlice({
@@ -38,10 +38,11 @@ export const config = createSlice({
   } as {
     currentProjectPath: string | undefined;
     currentSitePath: string | undefined;
-    rawConfigs: Record<string, { raw: Record<string, any>; validated: Record<string, any> }>;
+    rawConfigs: Record<string, { raw: Record<string, any>; validated: ValidatedRawConfig }>;
     projects: Record<string, ProjectConfig>;
     sites: Record<string, Record<string, any>>;
     filenames: Record<string, string>;
+    configExtensions?: string[];
   },
   reducers: {
     receiveCurrentProjectPath(state, action: PayloadAction<{ path: string }>) {
@@ -54,7 +55,7 @@ export const config = createSlice({
       state,
       action: PayloadAction<{
         raw: Record<string, any>;
-        validated: Record<string, any>;
+        validated: ValidatedRawConfig;
         path: string;
         file: string;
       }>,
@@ -71,6 +72,10 @@ export const config = createSlice({
       const { path, ...payload } = action.payload;
       state.projects[resolve(path)] = payload;
     },
+    receiveConfigExtension(state, action: PayloadAction<{ file: string }>) {
+      state.configExtensions ??= [];
+      state.configExtensions.push(action.payload.file);
+    },
   },
 });
 
diff --git a/packages/myst-cli/src/store/selectors.ts b/packages/myst-cli/src/store/selectors.ts
index 45b400aca..34f2ba053 100644
--- a/packages/myst-cli/src/store/selectors.ts
+++ b/packages/myst-cli/src/store/selectors.ts
@@ -2,7 +2,7 @@ import { resolve } from 'node:path';
 import type { ProjectConfig, SiteConfig } from 'myst-config';
 import type { LocalProject, LocalProjectPage } from '../project/types.js';
 import type { RootState } from './reducers.js';
-import type { BuildWarning, ExternalLinkResult } from './types.js';
+import type { BuildWarning, ExternalLinkResult, ValidatedRawConfig } from './types.js';
 
 function mutableCopy(obj?: Record<string, any>) {
   if (!obj) return;
@@ -65,10 +65,14 @@ export function selectLocalConfigFile(state: RootState, path: string): string |
 export function selectLocalRawConfig(
   state: RootState,
   path: string,
-): { raw: Record<string, any>; validated: Record<string, any> } | undefined {
+): { raw: Record<string, any>; validated: ValidatedRawConfig } | undefined {
   return mutableCopy(state.local.config.rawConfigs[resolve(path)]);
 }
 
+export function selectConfigExtensions(state: RootState): string[] {
+  return [...(state.local.config.configExtensions ?? [])];
+}
+
 export function selectReloadingState(state: RootState) {
   const { reloading, reloadRequested } = state.local.watch;
   return { reloading, reloadRequested };
diff --git a/packages/myst-cli/src/store/types.ts b/packages/myst-cli/src/store/types.ts
index 5077ec1cf..a772eaa8b 100644
--- a/packages/myst-cli/src/store/types.ts
+++ b/packages/myst-cli/src/store/types.ts
@@ -18,3 +18,9 @@ export type BuildWarning = {
   position?: VFileMessage['position'];
   ruleId?: string | null;
 };
+
+export type ValidatedRawConfig = {
+  site?: Record<string, any>;
+  project?: Record<string, any>;
+  extend?: string[];
+};

From 68c49298db5f6557a80a75ce3e37df45398d7e53 Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Thu, 16 May 2024 22:26:06 -0600
Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=A7=AA=20Extending=20config=20test?=
 =?UTF-8?q?=20case?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 packages/mystmd/tests/exports.yml             |  8 +++
 .../tests/extend-config/proj-a/index.md       |  3 +
 .../tests/extend-config/proj-a/myst.yml       | 15 +++++
 .../tests/extend-config/proj-b/myst.yml       | 20 +++++++
 .../tests/extend-config/proj-c/macros.yml     |  6 ++
 .../tests/outputs/extend-config-config.json   | 34 +++++++++++
 .../tests/outputs/extend-config-index.json    | 60 +++++++++++++++++++
 7 files changed, 146 insertions(+)
 create mode 100644 packages/mystmd/tests/extend-config/proj-a/index.md
 create mode 100644 packages/mystmd/tests/extend-config/proj-a/myst.yml
 create mode 100644 packages/mystmd/tests/extend-config/proj-b/myst.yml
 create mode 100644 packages/mystmd/tests/extend-config/proj-c/macros.yml
 create mode 100644 packages/mystmd/tests/outputs/extend-config-config.json
 create mode 100644 packages/mystmd/tests/outputs/extend-config-index.json

diff --git a/packages/mystmd/tests/exports.yml b/packages/mystmd/tests/exports.yml
index ab71a1a88..89c11945f 100644
--- a/packages/mystmd/tests/exports.yml
+++ b/packages/mystmd/tests/exports.yml
@@ -144,3 +144,11 @@ cases:
     outputs:
       - path: notebook-fig-embed/_build/site/content/index.json
         content: notebook-fig-embed/outputs/index.json
+  - title: Extend config test
+    cwd: extend-config/proj-a
+    command: myst build
+    outputs:
+      - path: extend-config/proj-a/_build/site/content/index.json
+        content: outputs/extend-config-index.json
+      - path: extend-config/proj-a/_build/site/config.json
+        content: outputs/extend-config-config.json
diff --git a/packages/mystmd/tests/extend-config/proj-a/index.md b/packages/mystmd/tests/extend-config/proj-a/index.md
new file mode 100644
index 000000000..0e80b7d19
--- /dev/null
+++ b/packages/mystmd/tests/extend-config/proj-a/index.md
@@ -0,0 +1,3 @@
+# Testing Config
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
\ No newline at end of file
diff --git a/packages/mystmd/tests/extend-config/proj-a/myst.yml b/packages/mystmd/tests/extend-config/proj-a/myst.yml
new file mode 100644
index 000000000..dee5d27c7
--- /dev/null
+++ b/packages/mystmd/tests/extend-config/proj-a/myst.yml
@@ -0,0 +1,15 @@
+version: 1
+project:
+  id: 7c806d98-0093-41ae-9889-dea2b9019064
+  title: MyST Config Extend Test
+  math:
+    a: defined-in-project-a
+    b: defined-in-project-a
+site:
+  template: ../../templates/site/myst/book-theme
+  nav: []
+  actions:
+    - title: Learn More
+      url: https://mystmd.org/guide
+  domains: []
+extends: ../proj-b/myst.yml
diff --git a/packages/mystmd/tests/extend-config/proj-b/myst.yml b/packages/mystmd/tests/extend-config/proj-b/myst.yml
new file mode 100644
index 000000000..b18234577
--- /dev/null
+++ b/packages/mystmd/tests/extend-config/proj-b/myst.yml
@@ -0,0 +1,20 @@
+version: 1
+project:
+  id: 36470d68-33d2-4b16-b0cf-5c955396fe57
+  keywords:
+    - my-kw
+  authors:
+    - John Doe
+  license: CC-BY-4.0
+  math:
+    a: defined-in-project-b
+    c: defined-in-project-b
+site:
+  template: book-theme
+  nav: []
+  actions:
+    - title: Learn More
+      url: https://mystmd.org/guide
+  domains: []
+extends:
+  - ../proj-c/macros.yml
diff --git a/packages/mystmd/tests/extend-config/proj-c/macros.yml b/packages/mystmd/tests/extend-config/proj-c/macros.yml
new file mode 100644
index 000000000..60700801f
--- /dev/null
+++ b/packages/mystmd/tests/extend-config/proj-c/macros.yml
@@ -0,0 +1,6 @@
+version: 1
+project:
+  math:
+    a: defined-in-project-c
+    c: defined-in-project-c
+    d: defined-in-project-c
diff --git a/packages/mystmd/tests/outputs/extend-config-config.json b/packages/mystmd/tests/outputs/extend-config-config.json
new file mode 100644
index 000000000..cd4c597cb
--- /dev/null
+++ b/packages/mystmd/tests/outputs/extend-config-config.json
@@ -0,0 +1,34 @@
+{
+  "options": {},
+  "nav": [],
+  "actions": [
+    { "title": "Learn More", "url": "https://mystmd.org/guide", "internal": false, "static": false }
+  ],
+  "projects": [
+    {
+      "license": {
+        "content": {
+          "id": "CC-BY-4.0",
+          "name": "Creative Commons Attribution 4.0 International",
+          "free": true,
+          "CC": true,
+          "url": "https://creativecommons.org/licenses/by/4.0/"
+        }
+      },
+      "math": {
+        "a": "defined-in-project-a",
+        "c": "defined-in-project-b",
+        "d": "defined-in-project-c",
+        "b": "defined-in-project-a"
+      },
+      "title": "MyST Config Extend Test",
+      "authors": [{ "id": "John Doe", "name": "John Doe" }],
+      "keywords": ["my-kw"],
+      "id": "7c806d98-0093-41ae-9889-dea2b9019064",
+      "exports": [],
+      "bibliography": [],
+      "index": "index",
+      "pages": []
+    }
+  ]
+}
diff --git a/packages/mystmd/tests/outputs/extend-config-index.json b/packages/mystmd/tests/outputs/extend-config-index.json
new file mode 100644
index 000000000..e9910c6ee
--- /dev/null
+++ b/packages/mystmd/tests/outputs/extend-config-index.json
@@ -0,0 +1,60 @@
+{
+  "kind": "Article",
+  "sha256": "b956ef955e38d755ee949dded619a8cbcd7dfbe7e33d368ab867a1aab49992ee",
+  "slug": "index",
+  "location": "/index.md",
+  "dependencies": [],
+  "frontmatter": {
+    "title": "Testing Config",
+    "content_includes_title": false,
+    "authors": [{ "id": "John Doe", "name": "John Doe" }],
+    "license": {
+      "content": {
+        "id": "CC-BY-4.0",
+        "name": "Creative Commons Attribution 4.0 International",
+        "free": true,
+        "CC": true,
+        "url": "https://creativecommons.org/licenses/by/4.0/"
+      }
+    },
+    "keywords": ["my-kw"],
+    "math": {
+      "a": "defined-in-project-a",
+      "c": "defined-in-project-b",
+      "d": "defined-in-project-c",
+      "b": "defined-in-project-a"
+    },
+    "exports": [
+      {
+        "format": "md",
+        "filename": "index.md",
+        "url": "/index-552a182d983b86e4781eb9df6c156089.md"
+      }
+    ]
+  },
+  "mdast": {
+    "type": "root",
+    "children": [
+      {
+        "type": "block",
+        "children": [
+          {
+            "type": "paragraph",
+            "position": { "start": { "line": 3, "column": 1 }, "end": { "line": 3, "column": 1 } },
+            "children": [
+              {
+                "type": "text",
+                "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
+                "position": {
+                  "start": { "line": 3, "column": 1 },
+                  "end": { "line": 3, "column": 1 }
+                }
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "references": { "cite": { "order": [], "data": {} } }
+}

From 200507027b5423d5a7a47f8d5008f93761287c1d Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Thu, 16 May 2024 23:37:01 -0600
Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=93=9A=20Small=20tweak=20to=20quick?=
 =?UTF-8?q?start=20docs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/quickstart-myst-websites.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/quickstart-myst-websites.md b/docs/quickstart-myst-websites.md
index a459471c5..dd5b99642 100644
--- a/docs/quickstart-myst-websites.md
+++ b/docs/quickstart-myst-websites.md
@@ -246,7 +246,7 @@ site:
   ...
 ```
 
-Doing this will keep the `_build` directory at the root level, but everything else outside of the `content` folder will be ignored. If you have a project in the same configuration file it can be accessed with `path: .`. Projects are "mounted" at the `slug:` (i.e. `/my-content/`) above.
+Doing this will keep the `_build` directory at the root level, but everything else outside of the `content` folder will be ignored. If you have a project in the same configuration file it can be accessed with `path: .`. Projects are "mounted" at the `slug:` (e.g. `/my-content/` above).
 :::
 
 ## Additional options

From fbbdfa5ded1e6d04fdb5d915e7cff8e6c0cdec93 Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Thu, 16 May 2024 23:37:23 -0600
Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=93=9A=20Add=20note=20about=20exten?=
 =?UTF-8?q?ds=20functionality=20to=20the=20docs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/frontmatter.md | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/frontmatter.md b/docs/frontmatter.md
index e0cd9ac19..9a2a51dc9 100644
--- a/docs/frontmatter.md
+++ b/docs/frontmatter.md
@@ -54,6 +54,25 @@ project:
   open_access: true
 ```
 
+(composing-myst-yml)=
+:::{note} Composing multiple `.yml` files
+
+You may separate your frontmatter into multiple, composable files. To reference other files from your main `myst.yml` file, use the `extends` key with relative path(s) to the other configuration files:
+
+```yaml
+version: 1
+site: ...
+project: ...
+extends:
+  - ../macros.yml
+  - ../funding.yml
+```
+
+Each of these files listed under `extends` must contain valid `myst.yml` structure with `version: 1` and `site` or `project` keys. They may also have additional files listed under `extends`.
+
+Composing files together this way allows you to have a single source of truth for project frontmatter that may be reused across multiple projects, for example math macros or funding information.
+:::
+
 +++
 
 ## Available frontmatter fields

From 19d66e529612fd4ac301a61c92eaa37c2f4a4957 Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Mon, 20 May 2024 01:13:09 -0600
Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=A7=AA=20Add=20page=20frontmatter?=
 =?UTF-8?q?=20to=20extend-config=20test?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 packages/mystmd/tests/extend-config/proj-a/index.md |  5 +++++
 .../mystmd/tests/outputs/extend-config-index.json   | 13 ++++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/mystmd/tests/extend-config/proj-a/index.md b/packages/mystmd/tests/extend-config/proj-a/index.md
index 0e80b7d19..8efc74893 100644
--- a/packages/mystmd/tests/extend-config/proj-a/index.md
+++ b/packages/mystmd/tests/extend-config/proj-a/index.md
@@ -1,3 +1,8 @@
+---
+math:
+  b: defined-on-page
+---
+
 # Testing Config
 
 Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
\ No newline at end of file
diff --git a/packages/mystmd/tests/outputs/extend-config-index.json b/packages/mystmd/tests/outputs/extend-config-index.json
index e9910c6ee..ada9751fc 100644
--- a/packages/mystmd/tests/outputs/extend-config-index.json
+++ b/packages/mystmd/tests/outputs/extend-config-index.json
@@ -1,6 +1,6 @@
 {
   "kind": "Article",
-  "sha256": "b956ef955e38d755ee949dded619a8cbcd7dfbe7e33d368ab867a1aab49992ee",
+  "sha256": "1303d3b24843b9c9a7a3df04f2b8625a9e690eae98dee1f87ee870d6db6986af",
   "slug": "index",
   "location": "/index.md",
   "dependencies": [],
@@ -22,13 +22,12 @@
       "a": "defined-in-project-a",
       "c": "defined-in-project-b",
       "d": "defined-in-project-c",
-      "b": "defined-in-project-a"
+      "b": "defined-on-page"
     },
     "exports": [
       {
         "format": "md",
-        "filename": "index.md",
-        "url": "/index-552a182d983b86e4781eb9df6c156089.md"
+        "filename": "index.md"
       }
     ]
   },
@@ -40,14 +39,14 @@
         "children": [
           {
             "type": "paragraph",
-            "position": { "start": { "line": 3, "column": 1 }, "end": { "line": 3, "column": 1 } },
+            "position": { "start": { "line": 8, "column": 1 }, "end": { "line": 8, "column": 1 } },
             "children": [
               {
                 "type": "text",
                 "value": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
                 "position": {
-                  "start": { "line": 3, "column": 1 },
-                  "end": { "line": 3, "column": 1 }
+                  "start": { "line": 8, "column": 1 },
+                  "end": { "line": 8, "column": 1 }
                 }
               }
             ]

From 49882d5641b9a266b8405f339e8339eb549b56ec Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Mon, 20 May 2024 01:16:32 -0600
Subject: [PATCH 10/11] =?UTF-8?q?=F0=9F=93=9A=20Move=20composing-myst-yml?=
 =?UTF-8?q?=20from=20note=20to=20section=20in=20docs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/frontmatter.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/frontmatter.md b/docs/frontmatter.md
index 9a2a51dc9..c3626c30a 100644
--- a/docs/frontmatter.md
+++ b/docs/frontmatter.md
@@ -55,7 +55,7 @@ project:
 ```
 
 (composing-myst-yml)=
-:::{note} Composing multiple `.yml` files
+#### Composing multiple `.yml` files
 
 You may separate your frontmatter into multiple, composable files. To reference other files from your main `myst.yml` file, use the `extends` key with relative path(s) to the other configuration files:
 
@@ -71,7 +71,6 @@ extends:
 Each of these files listed under `extends` must contain valid `myst.yml` structure with `version: 1` and `site` or `project` keys. They may also have additional files listed under `extends`.
 
 Composing files together this way allows you to have a single source of truth for project frontmatter that may be reused across multiple projects, for example math macros or funding information.
-:::
 
 +++
 

From 255c13bdec723abf2c0b68cfac2306e96e88bdeb Mon Sep 17 00:00:00 2001
From: Franklin Koch <franklinwkoch@gmail.com>
Date: Tue, 21 May 2024 14:17:15 -0600
Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=A7=B9=20Tidy=20frontmatter=20test?=
 =?UTF-8?q?=20data?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../myst-frontmatter/src/utils/fillPageFrontmatter.spec.ts     | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/myst-frontmatter/src/utils/fillPageFrontmatter.spec.ts b/packages/myst-frontmatter/src/utils/fillPageFrontmatter.spec.ts
index f126d5cad..b3caa5cf2 100644
--- a/packages/myst-frontmatter/src/utils/fillPageFrontmatter.spec.ts
+++ b/packages/myst-frontmatter/src/utils/fillPageFrontmatter.spec.ts
@@ -17,7 +17,6 @@ const TEST_PAGE_FRONTMATTER: PageFrontmatter = {
     },
   ],
   affiliations: [{ id: 'univb', name: 'University B' }],
-  name: 'example.md',
   doi: '10.1000/abcd/efg012',
   arxiv: 'https://arxiv.org/example',
   open_access: true,
@@ -52,7 +51,6 @@ const TEST_PROJECT_FRONTMATTER: ProjectFrontmatter = {
   ],
   affiliations: [{ id: 'univa', name: 'University A' }],
   date: '14 Dec 2021',
-  name: 'example.md',
   doi: '10.1000/abcd/efg012',
   arxiv: 'https://arxiv.org/example',
   open_access: true,
@@ -100,7 +98,6 @@ describe('fillPageFrontmatter', () => {
     const result = { ...TEST_PROJECT_FRONTMATTER };
     delete result.title;
     delete result.description;
-    delete result.name;
     delete result.oxa;
     delete result.exports;
     delete result.requirements;