Refactor: Rewrite of the Explorer to use ARIA trees

ts/a11y/SpeechUtil.ts

@@ -0,0 +1,232 @@
+/*************************************************************
+ *
+ *  Copyright (c) 2018-2023 The MathJax Consortium
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/**
+ * @fileoverview  Provides utility functions for speech handling.
+ *
+ * @author [email protected] (Volker Sorge)
+ */
+
+import {MmlNode} from '../core/MmlTree/MmlNode.js';
+import Sre from './sre.js';
+
+const ProsodyKeys = [ 'pitch', 'rate', 'volume' ];
+
+interface ProsodyElement {
+  [propName: string]: string | boolean | number;
+  pitch?: number;
+  rate?: number;
+  volume?: number;
+}
+
+export interface SsmlElement extends ProsodyElement {
+  pause?: string;
+  text?: string;
+  mark?: string;
+  character?: boolean;
+  kind?: string;
+}
+
+/**
+ * Parses a string containing an ssml structure into a list of text strings
+ * with associated ssml annotation elements.
+ *
+ * @param {string} speech The speech string.
+ * @return {[string, SsmlElement[]]} The annotation structure.
+ */
+export function ssmlParsing(speech: string): [string, SsmlElement[]] {
+  let xml = Sre.parseDOM(speech);
+  let instr: SsmlElement[] = [];
+  let text: String[] = [];
+  recurseSsml(Array.from(xml.childNodes), instr, text);
+  return [text.join(' '), instr];
+}
+
+/**
+ * Tail recursive combination of SSML components.
+ *
+ * @param {Node[]} nodes A list of SSML nodes.
+ * @param {SsmlElement[]} instr Accumulator for collating Ssml annotation
+ *    elements.
+ * @param {String[]} text A list of text elements.
+ * @param {ProsodyElement?} prosody The currently active prosody elements.
+ */
+function recurseSsml(nodes: Node[], instr: SsmlElement[], text: String[],
+                     prosody: ProsodyElement = {}) {
+  for (let node of nodes) {
+    if (node.nodeType === 3) {
+      let content = node.textContent.trim();
+      if (content) {
+        text.push(content);
+        instr.push(Object.assign({text: content}, prosody));
+      }
+      continue;
+    }
+    if (node.nodeType === 1) {
+      let element = node as Element;
+      let tag = element.tagName;
+      if (tag === 'speak') {
+        continue;
+      }
+      if (tag === 'prosody') {
+        recurseSsml(
+          Array.from(node.childNodes), instr, text,
+          getProsody(element, prosody));
+        continue;
+      }
+      switch (tag) {
+        case 'break':
+          instr.push({pause: element.getAttribute('time')});
+          break;
+        case 'mark':
+          instr.push({mark: element.getAttribute('name')});
+          break;
+        case 'say-as':
+          let txt = element.textContent;
+          instr.push(Object.assign({text: txt, character: true}, prosody));
+          text.push(txt);
+          break;
+      }
+    }
+  }
+}
+
+/**
+ * Maps prosody types to scaling functions.
+ */
+// TODO: These should be tweaked after more testing.
+const combinePros: {[key: string]: (x: number, sign: string) => number} = {
+  pitch: (x: number, _sign: string) => 1 * (x / 100),
+  volume: (x: number, _sign: string) => .5 * (x / 100),
+  rate: (x: number, _sign: string) =>  1 * (x / 100)
+};
+
+/**
+ * Retrieves prosody annotations from and SSML node.
+ * @param {Element} element The SSML node.
+ * @param {ProsodyElement} prosody The prosody annotation.
+ */
+function getProsody(element: Element, prosody: ProsodyElement) {
+  let combine: ProsodyElement = {};
+  for (let pros of ProsodyKeys) {
+    if (element.hasAttribute(pros)) {
+      let [sign, value] = extractProsody(element.getAttribute(pros));
+      if (!sign) {
+        // TODO: Sort out the base value. It is .5 for volume!
+        combine[pros] = (pros === 'volume') ? .5 : 1;
+        continue;
+      }
+      let orig = prosody[pros] as number;
+      orig = orig ? orig : ((pros === 'volume') ? .5 : 1);
+      let relative = combinePros[pros](parseInt(value, 10), sign);
+      combine[pros] = (sign === '-') ? orig - relative : orig + relative;
+    }
+  }
+  return combine;
+}
+
+/**
+ * Extracts the prosody value from an attribute.
+ */
+const prosodyRegexp = /([\+-]?)([0-9]+)%/;
+
+/**
+ * Extracts the prosody value from an attribute.
+ * @param {string} attr
+ */
+function extractProsody(attr: string) {
+  let match = attr.match(prosodyRegexp);
+  if (!match) {
+    console.warn('Something went wrong with the prosody matching.');
+    return ['', '100'];
+  }
+  return [match[1], match[2]];
+}
+
+/**
+ * Computes the aria-label from the node.
+ * @param {MmlNode} node The Math element.
+ * @param {string=} sep The speech separator. Defaults to space.
+ */
+function getLabel(node: MmlNode, sep: string = ' ') {
+  const attributes = node.attributes;
+  const speech = attributes.getExplicit('data-semantic-speech') as string;
+  if (!speech) {
+    return '';
+  }
+  const label = [speech];
+  const prefix = attributes.getExplicit('data-semantic-prefix') as string;
+  if (prefix) {
+    label.unshift(prefix);
+  }
+  // TODO: check if we need this or if it is automatic by the screen readers.
+  const postfix = attributes.getExplicit('data-semantic-postfix') as string;
+  if (postfix) {
+    label.push(postfix);
+  }
+  // TODO: Do we need to merge wrt. locale in SRE.
+  return label.join(sep);
+}
+
+/**
+ * Builds speechs from SSML markup strings.
+ *
+ * @param {string} speech The speech string.
+ * @param {string=} locale An optional locale.
+ * @param {string=} rate The base speech rate.
+ * @return {[string, SsmlElement[]]} The speech with the ssml annotation structure 
+ */
+export function buildSpeech(speech: string, locale: string = 'en',
+                            rate: string = '100'): [string, SsmlElement[]] {
+  return ssmlParsing('<?xml version="1.0"?><speak version="1.1"' +
+    ' xmlns="http://www.w3.org/2001/10/synthesis"' +
+    ` xml:lang="${locale}">` +
+    `<prosody rate="${rate}%">${speech}`+
+    '</prosody></speak>');
+}
+
+/**
+ * Retrieve and sets aria and braille labels recursively.
+ * @param {MmlNode} node The root node to search from.
+ */
+export function setAria(node: MmlNode, locale: string) {
+  const attributes = node.attributes;
+  if (!attributes) return;
+  const speech = getLabel(node);
+  if (speech) {
+    attributes.set('aria-label', buildSpeech(speech, locale)[0]);
+  }
+  const braille = node.attributes.getExplicit('data-semantic-braille') as string;
+  if (braille) {
+    attributes.set('aria-braillelabel', braille);
+  }
+  for (let child of node.childNodes) {
+    setAria(child, locale);
+  }
+}
+
+/**
+ * Creates a honking sound.
+ */
+export function honk() {
+  let ac = new AudioContext();
+  let os = ac.createOscillator();
+  os.frequency.value = 300;
+  os.connect(ac.destination);
+  os.start(ac.currentTime);
+  os.stop(ac.currentTime + .05);
+}

ts/a11y/explorer.ts

@@ -123,7 +123,7 @@ export function ExplorerMathItemMixin<B extends Constructor<HTMLMATHITEM>>(
         if (!this.explorers) {
           this.explorers = new ExplorerPool();
         }
-        this.explorers.init(document, node, mml);
+        this.explorers.init(document, node, mml, this);
       }
       this.state(STATE.EXPLORER);
     }
@@ -199,7 +199,9 @@ export function ExplorerMathDocumentMixin<B extends MathDocumentConstructor<HTML
       }),
       sre: expandable({
         ...BaseDocument.OPTIONS.sre,
-        speech: 'shallow',                 // overrides option in EnrichedMathDocument
+        speech: 'none',                    // None as speech is explicitly computed
+        structure: true,                   // Generates full aria structure
+        aria: true,
       }),
       a11y: {
         align: 'top',                      // placement of magnified expression
@@ -259,6 +261,7 @@ export function ExplorerMathDocumentMixin<B extends MathDocumentConstructor<HTML
      * @return {ExplorerMathDocument}   The MathDocument (so calls can be chained)
      */
     public explorable(): ExplorerMathDocument {
+      this.options.enableSpeech = true;
       if (!this.processed.isSet('explorer')) {
         if (this.options.enableExplorer) {
           if (!this.explorerRegions) {

ts/a11y/explorer/ExplorerPool.ts

@@ -23,7 +23,7 @@
  */
 
 import {LiveRegion, SpeechRegion, ToolTip, HoverRegion} from './Region.js';
-import type { ExplorerMathDocument } from '../explorer.js';
+import type { ExplorerMathDocument, ExplorerMathItem } from '../explorer.js';
 
 import {Explorer} from './Explorer.js';
 import * as ke from './KeyExplorer.js';
@@ -88,32 +88,11 @@ type ExplorerInit = (doc: ExplorerMathDocument, pool: ExplorerPool,
 let allExplorers: {[options: string]: ExplorerInit} = {
   speech: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => {
     let explorer = ke.SpeechExplorer.create(
-      doc, pool, doc.explorerRegions.speechRegion, node, ...rest) as ke.SpeechExplorer;
-    explorer.speechGenerator.setOptions({
-      automark: true as any, markup: 'ssml',
-      locale: doc.options.sre.locale, domain: doc.options.sre.domain,
-      style: doc.options.sre.style, modality: 'speech'});
-    // This weeds out the case of providing a non-existent locale option.
-    let locale = explorer.speechGenerator.getOptions().locale;
-    if (locale !== Sre.engineSetup().locale) {
-      doc.options.sre.locale = Sre.engineSetup().locale;
-      explorer.speechGenerator.setOptions({locale: doc.options.sre.locale});
-    }
+      doc, pool, doc.explorerRegions.speechRegion, node,
+      doc.explorerRegions.brailleRegion, doc.explorerRegions.magnifier, rest[0], rest[1]) as ke.SpeechExplorer;
     explorer.sound = true;
-    explorer.showRegion = 'subtitles';
-    return explorer;
-  },
-  braille: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) => {
-    let explorer = ke.SpeechExplorer.create(
-      doc, pool, doc.explorerRegions.brailleRegion, node, ...rest) as ke.SpeechExplorer;
-    explorer.speechGenerator.setOptions({automark: false as any, markup: 'none',
-                                         locale: 'nemeth', domain: 'default',
-                                         style: 'default', modality: 'braille'});
-    explorer.showRegion = 'viewBraille';
     return explorer;
   },
-  keyMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ...rest: any[]) =>
-    ke.Magnifier.create(doc, pool, doc.explorerRegions.magnifier, node, ...rest),
   mouseMagnifier: (doc: ExplorerMathDocument, pool: ExplorerPool, node: HTMLElement, ..._rest: any[]) =>
     me.ContentHoverer.create(doc, pool, doc.explorerRegions.magnifier, node,
                              (x: HTMLElement) => x.hasAttribute('data-semantic-type'),
@@ -212,13 +191,14 @@ export class ExplorerPool {
    * @param  mml The corresponding Mathml node as a string.
    */
   public init(document: ExplorerMathDocument,
-              node: HTMLElement, mml: string) {
+              node: HTMLElement, mml: string,
+              item: ExplorerMathItem) {
     this.document = document;
     this.mml = mml;
     this.node = node;
     this.setPrimaryHighlighter();
     for (let key of Object.keys(allExplorers)) {
-      this.explorers[key] = allExplorers[key](this.document, this, this.node, this.mml);
+      this.explorers[key] = allExplorers[key](this.document, this, this.node, this.mml, item);
     }
     this.setSecondaryHighlighter();
     this.attach();
@@ -233,7 +213,7 @@ export class ExplorerPool {
     let keyExplorers = [];
     for (let key of Object.keys(this.explorers)) {
       let explorer = this.explorers[key];
-      if (explorer instanceof ke.AbstractKeyExplorer) {
+      if (explorer instanceof ke.SpeechExplorer) {
         explorer.AddEvents();
         explorer.stoppable = false;
         keyExplorers.unshift(explorer);