🔧 Refactor myst-frontmatter (#734)

* 💣 Pull apart giant frontmatter file

* 🧪 Relocate frontmatter tests closer to code

* 🔧Share frontmatter aliases, improve validation naming

* 💣 Split tests out from giant files into smaller files

* 🕵️‍♀️ Remove aliases that do not resolve to valid key

* 📮 Postal code may be a number in frontmatter

* 🔧 Tweak how thebe is coerced when invalid for idempotency

* 🔧 Move jupytext/kernelspec frontmatter code

* 🔧 Move list coercing to simple-validators

* 🔧 Add missing exports

* 🧹 Clean up commented code

.changeset/large-laws-admire.md

@@ -0,0 +1,5 @@
+---
+'myst-frontmatter': patch
+---
+
+Postal code may be a number

.changeset/soft-fishes-applaud.md

@@ -0,0 +1,6 @@
+---
+'simple-validators': patch
+'myst-frontmatter': patch
+---
+
+Move list coercing to simple-validators

.changeset/spotty-dragons-move.md

@@ -0,0 +1,5 @@
+---
+'simple-validators': patch
+---
+
+Remove aliases that do not resolve to valid keys

.changeset/strange-kids-tap.md

@@ -0,0 +1,6 @@
+---
+'myst-frontmatter': patch
+'myst-config': patch
+---
+
+Share frontmatter aliases, improve validation naming

packages/myst-config/src/project/validators.ts

@@ -1,4 +1,8 @@
-import { PROJECT_FRONTMATTER_KEYS, validateProjectFrontmatterKeys } from 'myst-frontmatter';
+import {
+  FRONTMATTER_ALIASES,
+  PROJECT_FRONTMATTER_KEYS,
+  validateProjectFrontmatterKeys,
+} from 'myst-frontmatter';
 import type { ValidationOptions } from 'simple-validators';
 import {
   defined,
@@ -10,13 +14,8 @@ import {
 import type { ProjectConfig } from './types.js';
 
 const PROJECT_CONFIG_KEYS = {
-  optional: ['remote', 'index', 'exclude', 'plugins'].concat(PROJECT_FRONTMATTER_KEYS),
-  alias: {
-    jupyter: 'thebe',
-    author: 'authors',
-    affiliation: 'affiliations',
-    export: 'exports',
-  },
+  optional: ['remote', 'index', 'exclude', 'plugins', ...PROJECT_FRONTMATTER_KEYS],
+  alias: FRONTMATTER_ALIASES,
 };
 
 function validateProjectConfigKeys(value: Record<string, any>, opts: ValidationOptions) {

packages/myst-config/src/site/types.ts

@@ -1,5 +1,4 @@
 import type { Export, ProjectFrontmatter, SiteFrontmatter } from 'myst-frontmatter';
-import { SITE_FRONTMATTER_KEYS } from 'myst-frontmatter';
 
 export interface SiteProject {
   slug?: string;
@@ -23,12 +22,6 @@ export interface SiteAction {
   static?: boolean;
 }
 
-export const SITE_CONFIG_KEYS = {
-  optional: ['projects', 'nav', 'actions', 'domains', 'favicon', 'template'].concat(
-    SITE_FRONTMATTER_KEYS,
-  ),
-};
-
 export type SiteTemplateOptions = Record<string, any>;
 
 export type SiteConfig = SiteFrontmatter & {

packages/myst-config/src/site/validators.ts

@@ -12,8 +12,11 @@ import {
   validateBoolean,
   validationError,
 } from 'simple-validators';
-import { validateSiteFrontmatterKeys } from 'myst-frontmatter';
-import { SITE_CONFIG_KEYS } from './types.js';
+import {
+  FRONTMATTER_ALIASES,
+  SITE_FRONTMATTER_KEYS,
+  validateSiteFrontmatterKeys,
+} from 'myst-frontmatter';
 import type {
   SiteAction,
   SiteConfig,
@@ -22,6 +25,19 @@ import type {
   SiteTemplateOptions,
 } from './types.js';
 
+export const SITE_CONFIG_KEYS = {
+  optional: [
+    ...SITE_FRONTMATTER_KEYS,
+    'projects',
+    'nav',
+    'actions',
+    'domains',
+    'favicon',
+    'template',
+  ],
+  alias: FRONTMATTER_ALIASES,
+};
+
 function validateUrlOrPath(input: any, opts: ValidationOptions) {
   const value = validateString(input, opts);
   if (!defined(value)) return undefined;

packages/myst-frontmatter/package.json

@@ -40,6 +40,7 @@
   },
   "devDependencies": {
     "@types/spdx-correct": "^3.1.0",
+    "glob": "^10.3.1",
     "js-yaml": "^4.1.0",
     "moment": "^2.29.4"
   }

packages/myst-frontmatter/tests/affiliations.yml → packages/myst-frontmatter/src/affiliations/affiliations.yml

@@ -329,3 +329,74 @@ cases:
         - id: univa
           name: University A
     errors: 1
+
+  - title: extra keys removed
+    raw:
+      affiliations:
+        - name: University A
+          id: univa
+          extra: ''
+    normalized:
+      affiliations:
+        - name: University A
+          id: univa
+    warnings: 1
+  - title: full object returns self
+    raw:
+      affiliations:
+        - id: abc123
+          address: 123 Example St.
+          city: Example town
+          state: EX
+          postal_code: '00000'
+          country: USA
+          name: Example University
+          department: Example department
+          collaboration: true
+          isni: '0000000000000000'
+          ringgold: 99999
+          ror: '0000000000000000'
+          url: http://example.com
+          email: [email protected]
+          phone: 1-800-000-0000
+          fax: (800) 000-0001
+    normalized:
+      affiliations:
+        - id: abc123
+          address: 123 Example St.
+          city: Example town
+          state: EX
+          postal_code: '00000'
+          country: USA
+          name: Example University
+          department: Example department
+          collaboration: true
+          isni: '0000000000000000'
+          ringgold: 99999
+          ror: '0000000000000000'
+          url: http://example.com
+          email: [email protected]
+          phone: 1-800-000-0000
+          fax: (800) 000-0001
+  - title: invalid ringgold number errors
+    raw:
+      affiliations:
+        - name: University A
+          id: univa
+          ringgold: 1
+    normalized:
+      affiliations:
+        - name: University A
+          id: univa
+    errors: 1
+  - title: numeric postal code coerces to string
+    raw:
+      affiliations:
+        - name: University A
+          id: univa
+          postal_code: 91104
+    normalized:
+      affiliations:
+        - name: University A
+          id: univa
+          postal_code: '91104'

packages/myst-frontmatter/src/affiliations/types.ts

@@ -0,0 +1,20 @@
+export interface Affiliation {
+  id?: string;
+  name?: string; // by default required but if only institution is provided, it's ok
+  institution?: string;
+  department?: string;
+  address?: string;
+  city?: string;
+  state?: string; // or region or province
+  postal_code?: string;
+  country?: string;
+  collaboration?: boolean;
+  isni?: string;
+  ringgold?: number;
+  ror?: string;
+  doi?: string;
+  url?: string;
+  email?: string;
+  phone?: string;
+  fax?: string;
+}

packages/myst-frontmatter/src/affiliations/validators.ts

@@ -0,0 +1,131 @@
+import type { ValidationOptions } from 'simple-validators';
+import {
+  defined,
+  incrementOptions,
+  validateBoolean,
+  validateEmail,
+  validateNumber,
+  validateObjectKeys,
+  validateString,
+  validateUrl,
+  validationWarning,
+} from 'simple-validators';
+import { stashPlaceholder } from '../utils/referenceStash.js';
+import { validateDoi } from '../utils/validators.js';
+import type { Affiliation } from './types.js';
+
+const AFFILIATION_KEYS = [
+  'id',
+  'address',
+  'city',
+  'state',
+  'postal_code',
+  'country',
+  'name',
+  'department',
+  'collaboration',
+  'isni',
+  'ringgold',
+  'ror',
+  'doi',
+  'url',
+  'email',
+  'phone',
+  'fax',
+];
+
+const AFFILIATION_ALIASES = {
+  ref: 'id', // Used in QMD to reference an affiliation
+  region: 'state',
+  province: 'state',
+  zipcode: 'postal_code',
+  zip_code: 'postal_code',
+  website: 'url',
+  institution: 'name',
+};
+
+/**
+ * Validate Affiliation object against the schema
+ */
+export function validateAffiliation(input: any, opts: ValidationOptions) {
+  if (typeof input === 'string') {
+    input = stashPlaceholder(input);
+  }
+  const value = validateObjectKeys(
+    input,
+    { optional: AFFILIATION_KEYS, alias: AFFILIATION_ALIASES },
+    opts,
+  );
+  if (value === undefined) return undefined;
+  const output: Affiliation = {};
+  if (defined(value.id)) {
+    output.id = validateString(value.id, incrementOptions('id', opts));
+  }
+  if (defined(value.name)) {
+    output.name = validateString(value.name, incrementOptions('name', opts));
+  }
+  if (defined(value.department)) {
+    output.department = validateString(value.department, incrementOptions('department', opts));
+  }
+  if (defined(value.address)) {
+    output.address = validateString(value.address, incrementOptions('address', opts));
+  }
+  if (defined(value.city)) {
+    output.city = validateString(value.city, incrementOptions('city', opts));
+  }
+  if (defined(value.state)) {
+    output.state = validateString(value.state, incrementOptions('state', opts));
+  }
+  if (defined(value.postal_code)) {
+    output.postal_code = validateString(value.postal_code, {
+      coerceNumber: true,
+      ...incrementOptions('postal_code', opts),
+    });
+  }
+  if (defined(value.country)) {
+    output.country = validateString(value.country, incrementOptions('country', opts));
+  }
+  // Both ISNI and ROR validation should occur similar to orcid (maybe in that same lib?)
+  if (defined(value.isni)) {
+    output.isni = validateString(value.isni, incrementOptions('isni', opts));
+  }
+  if (defined(value.ror)) {
+    output.ror = validateString(value.ror, incrementOptions('ror', opts));
+  }
+  if (defined(value.ringgold)) {
+    output.ringgold = validateNumber(value.ringgold, {
+      min: 1000,
+      max: 999999,
+      ...incrementOptions('ringgold', opts),
+    });
+  }
+  if (defined(value.doi)) {
+    output.doi = validateDoi(value.doi, incrementOptions('doi', opts));
+  }
+  if (defined(value.collaboration)) {
+    output.collaboration = validateBoolean(
+      value.collaboration,
+      incrementOptions('collaboration', opts),
+    );
+  }
+  if (defined(value.email)) {
+    output.email = validateEmail(value.email, incrementOptions('email', opts));
+  }
+  if (defined(value.url)) {
+    output.url = validateUrl(value.url, incrementOptions('url', opts));
+  }
+  if (defined(value.phone)) {
+    output.phone = validateString(value.phone, incrementOptions('phone', opts));
+  }
+  if (defined(value.fax)) {
+    output.fax = validateString(value.fax, incrementOptions('fax', opts));
+  }
+  // If affiliation only has an id, give it a matching name; this is equivalent to the case
+  // where a simple string is provided as an affiliation.
+  if (Object.keys(output).length === 1 && output.id) {
+    return stashPlaceholder(output.id);
+  } else if (!output.name) {
+    validationWarning('affiliation should include name/institution', opts);
+  }
+  return output;
+}

packages/myst-frontmatter/src/biblio/biblio.yml

@@ -0,0 +1,27 @@
+title: Biblio
+cases:
+  - title: empty object returns self
+    raw:
+      biblio: {}
+    normalized:
+      biblio: {}
+  - title: extra keys removed
+    raw:
+      biblio:
+        extra: ''
+    normalized:
+      biblio: {}
+    warnings: 1
+  - title: full object returns self
+    raw:
+      biblio:
+        volume: test
+        issue: example
+        first_page: 1
+        last_page: 2
+    normalized:
+      biblio:
+        volume: test
+        issue: example
+        first_page: 1
+        last_page: 2

packages/myst-frontmatter/src/biblio/index.ts

@@ -0,0 +1,2 @@
+export * from './types.js';
+export * from './validators.js';

packages/myst-frontmatter/src/biblio/types.ts

@@ -0,0 +1,7 @@
+export type Biblio = {
+  // https://docs.openalex.org/about-the-data/work#biblio
+  volume?: string | number; // sometimes you'll get fun values like "Spring" and "Inside cover."
+  issue?: string | number;
+  first_page?: string | number;
+  last_page?: string | number;
+};