Merge mobile and desktop modules output, replace article templating t…

…o interim DesktopRenderer and MobileRenderer (partial impl)
openzim · Oct 2, 2023 · 1865aaf · 1865aaf
1 parent 121f215
commit 1865aaf
Show file tree

Hide file tree

Showing 12 changed files with 154 additions and 112 deletions.
diff --git a/package.json b/package.json
@@ -15,6 +15,7 @@
     "test:pattern": "node --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand --forceExit --detectOpenHandles",
     "test:e2e": "npm run test:pattern 'test/e2e'",
     "test:unit": "npm run test:pattern 'test/unit'",
+    "test:mobile": "npm start -- --mwUrl=https://en.wikipedia.org [email protected] --articleList=Canada",
     "test:e2e-coverage": "npm run test:pattern 'test/e2e' --  --coverage",
     "test:unit-coverage": "npm run test:pattern 'test/unit' --  --coverage",
     "test": "npm run test:unit-coverage -- --silent && npm run test:e2e-coverage -- --silent",

diff --git a/src/Downloader.ts b/src/Downloader.ts
@@ -714,10 +714,8 @@ class Downloader {
     }
     return {
       jsConfigVars,
-      jsDependenciesList,
-      styleDependenciesList,
-      mobileJsDependenciesList: this.mobileJsDependenciesList,
-      mobileStyleDependenciesList: this.mobileStyleDependenciesList,
+      jsDependenciesList: jsDependenciesList.concat(this.mobileJsDependenciesList),
+      styleDependenciesList: styleDependenciesList.concat(this.mobileStyleDependenciesList),
     }
   }
 

diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts
@@ -427,31 +427,17 @@ async function execute(argv: any) {
 
     logger.log('Getting articles')
     stime = Date.now()
-    const { jsModuleDependencies, cssModuleDependencies, jsMobileModuleDependencies, cssMobileModuleDependencies } = await saveArticles(
-      zimCreator,
-      downloader,
-      dump,
-      hasWikimediaMobileApi,
-      forceRender,
-    )
+    const { jsModuleDependencies, cssModuleDependencies } = await saveArticles(zimCreator, downloader, dump, hasWikimediaMobileApi, forceRender)
     logger.log(`Fetching Articles finished in ${(Date.now() - stime) / 1000} seconds`)
 
     logger.log(`Found [${jsModuleDependencies.size}] js module dependencies`)
     logger.log(`Found [${cssModuleDependencies.size}] style module dependencies`)
 
-    let allDependenciesWithType = [
+    const allDependenciesWithType = [
       { type: 'js', moduleList: Array.from(jsModuleDependencies) },
       { type: 'css', moduleList: Array.from(cssModuleDependencies) },
     ]
 
-    if (dump.opts.isMobileRenderer) {
-      allDependenciesWithType = [
-        ...allDependenciesWithType,
-        { type: 'mobileJs', moduleList: Array.from(jsMobileModuleDependencies) },
-        { type: 'mobileCss', moduleList: Array.from(cssMobileModuleDependencies) },
-      ]
-    }
-
     if (downloader.webp) {
       logger.log('Downloading polyfill module')
       importPolyfillModules(zimCreator)

diff --git a/src/renderers/abstract.renderer.ts b/src/renderers/abstract.renderer.ts
@@ -9,19 +9,14 @@ import DU from '../DOMUtils.js'
 import { config } from '../config.js'
 import { Dump } from '../Dump.js'
 import { rewriteUrlsOfDoc } from '../util/rewriteUrls.js'
-import { footerTemplate, htmlTemplateCode } from '../Templates.js'
+import { footerTemplate } from '../Templates.js'
 import {
   getFullUrl,
   getMediaBase,
   getMimeType,
   getRelativeFilePath,
   isWebpCandidateImageMimeType,
   interpolateTranslationString,
-  genCanonicalLink,
-  genHeaderScript,
-  genHeaderCSSLink,
-  genPCSOverrideScript,
-  genPCSCOverrideCSSLink,
   encodeArticleIdForZimHtmlUrl,
 } from '../util/misc.js'
 
@@ -388,7 +383,8 @@ export abstract class Renderer {
     return thumbDiv
   }
 
-  public async processHtml(html: string, dump: Dump, articleId: string, articleDetail: any, _moduleDependencies: any, webp: boolean) {
+  // TODO: The first part of this method is common for all renders
+  public async processHtml(html: string, dump: Dump, articleId: string, articleDetail: any, _moduleDependencies: any, webp: boolean, callback) {
     let mediaDependencies: Array<{ url: string; path: string }> = []
     let subtitles: Array<{ url: string; path: string }> = []
     let doc = domino.createDocument(html)
@@ -432,7 +428,8 @@ export abstract class Renderer {
       doc = await dump.customProcessor.preProcessArticle(articleId, doc)
     }
 
-    let templatedDoc = await this.templateArticle(doc, _moduleDependencies, dump, articleId, articleDetail, RedisStore.articleDetailXId)
+    let templatedDoc = callback(_moduleDependencies, articleId)
+    templatedDoc = await this.mergeTemplateDoc(templatedDoc, doc, dump, articleDetail, RedisStore.articleDetailXId, articleId)
 
     if (dump.customProcessor && dump.customProcessor.postProcessArticle) {
       templatedDoc = await dump.customProcessor.postProcessArticle(articleId, templatedDoc)
@@ -460,64 +457,14 @@ export abstract class Renderer {
     }
   }
 
-  private async templateArticle(
+  private async mergeTemplateDoc(
+    htmlTemplateDoc: DominoElement,
     parsoidDoc: DominoElement,
-    moduleDependencies: any,
     dump: Dump,
-    articleId: string,
     articleDetail: ArticleDetail,
     articleDetailXId: RKVS<ArticleDetail>,
-  ): Promise<Document> {
-    const { jsConfigVars, jsDependenciesList, styleDependenciesList, mobileJsDependenciesList, mobileStyleDependenciesList } = moduleDependencies as {
-      jsConfigVars: string | RegExpExecArray
-      jsDependenciesList: string[]
-      styleDependenciesList: string[]
-      mobileJsDependenciesList: string[]
-      mobileStyleDependenciesList: string[]
-    }
-
-    let htmlTemplateString = htmlTemplateCode(articleId).replace('__ARTICLE_CANONICAL_LINK__', genCanonicalLink(config, MediaWiki.webUrl.href, articleId))
-
-    if (!dump.opts.isMobileRenderer) {
-      htmlTemplateString = htmlTemplateString
-        .replace('__ARTICLE_CONFIGVARS_LIST__', jsConfigVars !== '' ? genHeaderScript(config, 'jsConfigVars', articleId, config.output.dirs.mediawiki) : '')
-        .replace(
-          '__ARTICLE_JS_LIST__',
-          jsDependenciesList.length !== 0 ? jsDependenciesList.map((oneJsDep) => genHeaderScript(config, oneJsDep, articleId, config.output.dirs.mediawiki)).join('\n') : '',
-        )
-        .replace(
-          '__ARTICLE_CSS_LIST__',
-          styleDependenciesList.length !== 0
-            ? styleDependenciesList.map((oneCssDep) => genHeaderCSSLink(config, oneCssDep, articleId, config.output.dirs.mediawiki)).join('\n')
-            : '',
-        )
-        .replace('__JS_SCRIPTS_MOBILE__', '')
-        .replace('__CSS_LINKS_MOBILE__', '')
-        .replace('__PCS_CSS_OVERRIDE__', '')
-        .replace('__PCS_JS_OVERRIDE__', '')
-    } else {
-      htmlTemplateString = htmlTemplateString
-        .replace('__ARTICLE_CONFIGVARS_LIST__', '')
-        .replace('__ARTICLE_JS_LIST__', '')
-        .replace('__ARTICLE_CSS_LIST__', '')
-        .replace(
-          '__JS_SCRIPTS_MOBILE__',
-          mobileJsDependenciesList.length !== 0
-            ? mobileJsDependenciesList.map((oneMobJsDep) => genHeaderScript(config, oneMobJsDep, articleId, config.output.dirs.mediawiki)).join('\n')
-            : '',
-        )
-        .replace(
-          '__CSS_LINKS_MOBILE__',
-          mobileStyleDependenciesList.length !== 0
-            ? mobileStyleDependenciesList.map((oneMobCssDep) => genHeaderCSSLink(config, oneMobCssDep, articleId, config.output.dirs.mediawiki)).join('\n')
-            : '',
-        )
-        .replace('__PCS_CSS_OVERRIDE__', genPCSCOverrideCSSLink(config.output.pcsCssResources[0]))
-        .replace('__PCS_JS_OVERRIDE__', genPCSOverrideScript(config.output.pcsJsResources[0]))
-    }
-
-    const htmlTemplateDoc = domino.createDocument(htmlTemplateString)
-
+    articleId: string,
+  ) {
     /* Create final document by merging template and parsoid documents */
     htmlTemplateDoc.getElementById('mw-content-text').style.setProperty('direction', dump.mwMetaData.textDir)
     htmlTemplateDoc.getElementById('mw-content-text').innerHTML = parsoidDoc.getElementsByTagName('body')[0].innerHTML

diff --git a/src/renderers/abstractDesktop.render.ts b/src/renderers/abstractDesktop.render.ts
@@ -0,0 +1,47 @@
+import * as domino from 'domino'
+import { Renderer } from './abstract.renderer.js'
+import { config } from '../config.js'
+import MediaWiki from '../MediaWiki.js'
+
+import { htmlTemplateCode } from '../Templates.js'
+import { genCanonicalLink, genHeaderScript, genHeaderCSSLink } from '../util/misc.js'
+
+export abstract class DesktopRenderer extends Renderer {
+  constructor() {
+    super()
+  }
+
+  public templateDesktopArticle(moduleDependencies: any, articleId: string): Document {
+    const { jsConfigVars, jsDependenciesList, styleDependenciesList } = moduleDependencies as {
+      jsConfigVars: string | RegExpExecArray
+      jsDependenciesList: string[]
+      styleDependenciesList: string[]
+    }
+
+    const desktopJsModuleDependencies = jsDependenciesList.filter((item) => !item.includes('javascript/mobile'))
+    const desktopCssModuleDependencies = styleDependenciesList.filter((item) => !item.includes('css/mobile'))
+
+    const htmlTemplateString = htmlTemplateCode(articleId)
+      .replace('__ARTICLE_CANONICAL_LINK__', genCanonicalLink(config, MediaWiki.webUrl.href, articleId))
+      .replace('__ARTICLE_CONFIGVARS_LIST__', jsConfigVars !== '' ? genHeaderScript(config, 'jsConfigVars', articleId, config.output.dirs.mediawiki) : '')
+      .replace(
+        '__ARTICLE_JS_LIST__',
+        desktopJsModuleDependencies.length !== 0
+          ? desktopJsModuleDependencies.map((oneJsDep) => genHeaderScript(config, oneJsDep, articleId, config.output.dirs.mediawiki)).join('\n')
+          : '',
+      )
+      .replace(
+        '__ARTICLE_CSS_LIST__',
+        desktopCssModuleDependencies.length !== 0
+          ? desktopCssModuleDependencies.map((oneCssDep) => genHeaderCSSLink(config, oneCssDep, articleId, config.output.dirs.mediawiki)).join('\n')
+          : '',
+      )
+      .replace('__JS_SCRIPTS_MOBILE__', '')
+      .replace('__CSS_LINKS_MOBILE__', '')
+      .replace('__PCS_CSS_OVERRIDE__', '')
+      .replace('__PCS_JS_OVERRIDE__', '')
+
+    const htmlTemplateDoc = domino.createDocument(htmlTemplateString)
+    return htmlTemplateDoc
+  }
+}
diff --git a/src/renderers/abstractMobile.render.ts b/src/renderers/abstractMobile.render.ts
@@ -0,0 +1,54 @@
+import * as domino from 'domino'
+import { Renderer } from './abstract.renderer.js'
+import { config } from '../config.js'
+import MediaWiki from '../MediaWiki.js'
+
+import { htmlTemplateCode } from '../Templates.js'
+import { genCanonicalLink, genHeaderScript, genHeaderCSSLink } from '../util/misc.js'
+
+export abstract class MobileRenderer extends Renderer {
+  constructor() {
+    super()
+  }
+
+  private genPCSCOverrideCSSLink(css: string) {
+    return `<link rel="stylesheet" href="../-/${css}.css" />`
+  }
+
+  private genPCSOverrideScript(js: string) {
+    return `<script src='../-/${js}.js'></script>`
+  }
+
+  public templateMobileArticle(moduleDependencies: any, articleId: string): Document {
+    const { jsDependenciesList, styleDependenciesList } = moduleDependencies as {
+      jsDependenciesList: string[]
+      styleDependenciesList: string[]
+    }
+
+    const mobileJsModuleDependencies = jsDependenciesList.filter((item) => item.includes('javascript/mobile'))
+    const mobileCssModuleDependencies = styleDependenciesList.filter((item) => item.includes('css/mobile'))
+
+    const htmlTemplateString = htmlTemplateCode(articleId)
+      .replace('__ARTICLE_CANONICAL_LINK__', genCanonicalLink(config, MediaWiki.webUrl.href, articleId))
+      .replace('__ARTICLE_CONFIGVARS_LIST__', '')
+      .replace('__ARTICLE_JS_LIST__', '')
+      .replace('__ARTICLE_CSS_LIST__', '')
+      .replace(
+        '__JS_SCRIPTS_MOBILE__',
+        mobileCssModuleDependencies.length !== 0
+          ? mobileCssModuleDependencies.map((oneMobJsDep) => genHeaderScript(config, oneMobJsDep, articleId, config.output.dirs.mediawiki)).join('\n')
+          : '',
+      )
+      .replace(
+        '__CSS_LINKS_MOBILE__',
+        mobileJsModuleDependencies.length !== 0
+          ? mobileJsModuleDependencies.map((oneMobCssDep) => genHeaderCSSLink(config, oneMobCssDep, articleId, config.output.dirs.mediawiki)).join('\n')
+          : '',
+      )
+      .replace('__PCS_CSS_OVERRIDE__', this.genPCSCOverrideCSSLink(config.output.pcsCssResources[0]))
+      .replace('__PCS_JS_OVERRIDE__', this.genPCSOverrideScript(config.output.pcsJsResources[0]))
+
+    const htmlTemplateDoc = domino.createDocument(htmlTemplateString)
+    return htmlTemplateDoc
+  }
+}
diff --git a/src/renderers/visual-editor.renderer.ts b/src/renderers/visual-editor.renderer.ts
@@ -1,6 +1,6 @@
-import { DELETED_ARTICLE_ERROR } from '../util/const.js'
 import * as logger from '../Logger.js'
-import { Renderer } from './abstract.renderer.js'
+import { DELETED_ARTICLE_ERROR } from '../util/const.js'
+import { DesktopRenderer } from './abstractDesktop.render.js'
 import { getStrippedTitleFromHtml } from '../util/misc.js'
 import { RenderOpts, RenderOutput } from './abstract.renderer.js'
 
@@ -9,7 +9,7 @@ Represent 'https://{wikimedia-wiki}/w/api.php?action=visualeditor&mobileformat=h
 or
 'https://{3rd-part-wikimedia-wiki}/w/api.php?action=visualeditor&mobileformat=html&format=json&paction=parse&page={title}'
 */
-export class VisualEditorRenderer extends Renderer {
+export class VisualEditorRenderer extends DesktopRenderer {
   constructor() {
     super()
   }
@@ -55,7 +55,15 @@ export class VisualEditorRenderer extends Renderer {
       const { articleId, articleDetail, webp, _moduleDependencies, dump } = renderOpts
       const { html, displayTitle } = await this.retrieveHtml(renderOpts)
       if (html) {
-        const { finalHTML, mediaDependencies, subtitles } = await super.processHtml(html, dump, articleId, articleDetail, _moduleDependencies, webp)
+        const { finalHTML, mediaDependencies, subtitles } = await super.processHtml(
+          html,
+          dump,
+          articleId,
+          articleDetail,
+          _moduleDependencies,
+          webp,
+          super.templateDesktopArticle(_moduleDependencies, articleId),
+        )
         result.push({
           articleId,
           displayTitle,

diff --git a/src/renderers/wikimedia-desktop.renderer.ts b/src/renderers/wikimedia-desktop.renderer.ts
@@ -1,10 +1,10 @@
 import domino from 'domino'
-import { Renderer } from './abstract.renderer.js'
+import { DesktopRenderer } from './abstractDesktop.render.js'
 import { getStrippedTitleFromHtml } from '../util/misc.js'
 import { RenderOpts, RenderOutput } from './abstract.renderer.js'
 
 // Represent 'https://{wikimedia-wiki}/api/rest_v1/page/html/'
-export class WikimediaDesktopRenderer extends Renderer {
+export class WikimediaDesktopRenderer extends DesktopRenderer {
   constructor() {
     super()
   }
@@ -47,7 +47,15 @@ export class WikimediaDesktopRenderer extends Renderer {
       if (!isMainPage) {
         dataWithHeader = super.injectH1TitleToHtml(data, articleDetail)
       }
-      const { finalHTML, mediaDependencies, subtitles } = await super.processHtml(dataWithHeader || data, dump, articleId, articleDetail, _moduleDependencies, webp)
+      const { finalHTML, mediaDependencies, subtitles } = await super.processHtml(
+        dataWithHeader || data,
+        dump,
+        articleId,
+        articleDetail,
+        _moduleDependencies,
+        webp,
+        super.templateDesktopArticle,
+      )
 
       result.push({
         articleId: _articleId,

diff --git a/src/renderers/wikimedia-mobile.renderer.ts b/src/renderers/wikimedia-mobile.renderer.ts
@@ -1,13 +1,13 @@
 import * as domino from 'domino'
 import * as logger from '../Logger.js'
-import { Renderer } from './abstract.renderer.js'
+import { MobileRenderer } from './abstractMobile.render.js'
 import { getStrippedTitleFromHtml } from '../util/misc.js'
 import { RenderOpts, RenderOutput } from './abstract.renderer.js'
 
 type PipeFunction = (value: DominoElement) => DominoElement | Promise<DominoElement>
 
 // Represent 'https://{wikimedia-wiki}/api/rest_v1/page/mobile-html/'
-export class WikimediaMobileRenderer extends Renderer {
+export class WikimediaMobileRenderer extends MobileRenderer {
   constructor() {
     super()
   }
@@ -37,7 +37,15 @@ export class WikimediaMobileRenderer extends Renderer {
           this.removeEditContainer,
           this.removeHiddenClass,
           async (doc) => {
-            const { finalHTML, subtitles, mediaDependencies } = await super.processHtml(doc.documentElement.outerHTML, dump, articleId, articleDetail, _moduleDependencies, webp)
+            const { finalHTML, subtitles, mediaDependencies } = await super.processHtml(
+              doc.documentElement.outerHTML,
+              dump,
+              articleId,
+              articleDetail,
+              _moduleDependencies,
+              webp,
+              super.templateMobileArticle,
+            )
 
             mediaDependenciesVal = mediaDependencies
             subtitlesVal = subtitles

diff --git a/src/util/dump.ts b/src/util/dump.ts
@@ -90,7 +90,7 @@ export async function getAndProcessStylesheets(downloader: Downloader, links: Ar
   })
 }
 
-export async function downloadAndSaveModule(zimCreator: ZimCreator, downloader: Downloader, dump: Dump, module: string, type: 'js' | 'css' | 'mobileJs' | 'mobileCss') {
+export async function downloadAndSaveModule(zimCreator: ZimCreator, downloader: Downloader, dump: Dump, module: string, type: 'js' | 'css') {
   const replaceCodeByRegex = (sourceText, replaceMap: Map<RegExp, string>) => {
     let text: string
     replaceMap.forEach((textToReplace, regEx) => {
@@ -118,16 +118,17 @@ export async function downloadAndSaveModule(zimCreator: ZimCreator, downloader:
 
   let apiParameterOnly
   let moduleApiUrl: string
-  if (type === 'js' || type === 'mobileJs') {
+  if (type === 'js') {
     apiParameterOnly = 'scripts'
-  } else if (type === 'css' || type === 'mobileCss') {
+  } else if (type === 'css') {
     apiParameterOnly = 'styles'
   }
 
   if (type === 'js' || type === 'css') {
     moduleApiUrl = encodeURI(`${MediaWiki.modulePath}debug=true&lang=en&modules=${module}&only=${apiParameterOnly}&skin=vector&version=&*`)
   } else if (type === 'mobileJs' || type === 'mobileCss') {
-    moduleApiUrl = encodeURI(`https:${module}`)
+    // TODO: apply condition to the mobile module Url
+    // moduleApiUrl = encodeURI(`https:${module}`)
   }
 
   logger.info(`Getting [${type}] module [${moduleApiUrl}]`)