Refactor mwApiPath (to mwActionApiPath), mwRestApiPath, mwModulePath and mwWikiPath parameters + fix regression of test render template

src/Downloader.ts

@@ -52,6 +52,7 @@ interface DownloaderOpts {
   s3?: S3
   webp: boolean
   backoffOptions?: BackoffOptions
+  mwWikiPath?: string
 }
 
 interface BackoffOptions {
@@ -107,7 +108,7 @@ class Downloader {
     this.optimisationCacheUrl = optimisationCacheUrl
     this.webp = webp
     this.s3 = s3
-    this.apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href)
+    this.apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)
 
     this.backoffOptions = {
       strategy: new backoff.ExponentialStrategy(),
@@ -620,7 +621,7 @@ class Downloader {
   }
 
   private async getSubCategories(articleId: string, continueStr = ''): Promise<Array<{ pageid: number; ns: number; title: string }>> {
-    const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href)
+    const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)
 
     const { query, continue: cont } = await this.getJSON<any>(apiUrlDirector.buildSubCategoriesURL(articleId, continueStr))
     const items = query.categorymembers.filter((a: any) => a && a.title)
@@ -651,7 +652,7 @@ class Downloader {
     let jsDependenciesList: string[] = []
     let styleDependenciesList: string[] = []
 
-    const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href)
+    const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)
 
     const articleApiUrl = apiUrlDirector.buildArticleApiURL(title)
 

src/MediaWiki.ts

@@ -44,20 +44,21 @@ class MediaWiki {
   public queryOpts: QueryOpts
 
   #wikiPath: string
-  #apiPath: string
+  #actionApiPath: string
+  #restApiPath: string
+  #modulePathOpt: string
   #username: string
   #password: string
-  #apiActionPath: string
   #domain: string
   private apiUrlDirector: ApiURLDirector
+  private baseUrlDirector: BaseURLDirector
   private wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
   private wikimediaMobileUrlDirector: WikimediaMobileURLDirector
-  private VisualEditorURLDirector: VisualEditorURLDirector
+  private visualEditorURLDirector: VisualEditorURLDirector
 
   public visualEditorApiUrl: URL
-  public apiUrl: URL
+  public actionApiUrl: URL
   public modulePath: string // only for reading
-  public _modulePathOpt: string // only for whiting to generate modulePath
   public mobileModulePath: string
   public webUrl: URL
   public WikimediaDesktopApiUrl: URL
@@ -76,29 +77,53 @@ class MediaWiki {
     this.#password = value
   }
 
-  set apiActionPath(value: string) {
-    this.#apiActionPath = value
+  set actionApiPath(value: string) {
+    if (value) {
+      this.#actionApiPath = value
+      this.initApiURLDirector()
+    }
   }
 
-  set apiPath(value: string) {
-    this.#apiPath = value
+  set restApiPath(value: string) {
+    if (value) {
+      this.#restApiPath = value
+      this.initApiURLDirector()
+    }
   }
 
   set domain(value: string) {
     this.#domain = value
   }
 
   set wikiPath(value: string) {
-    this.#wikiPath = value
+    if (value) {
+      this.#wikiPath = value
+      this.initApiURLDirector()
+    }
   }
 
   set base(value: string) {
-    this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value)
-    this.initMWApis()
+    if (value) {
+      this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value)
+      this.baseUrlDirector = new BaseURLDirector(this.baseUrl.href)
+      this.initMWApis()
+      this.initApiURLDirector()
+    }
   }
 
   set modulePathOpt(value: string) {
-    this._modulePathOpt = value
+    if (value) {
+      this.#modulePathOpt = value
+      if (this.baseUrlDirector) {
+        this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt)
+      } else {
+        logger.error('Base url director should be specified first')
+      }
+    } else {
+      if (this.baseUrlDirector) {
+        this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt)
+      }
+    }
   }
 
   private initializeMediaWikiDefaults(): void {
@@ -107,11 +132,13 @@ class MediaWiki {
     this.#password = ''
     this.getCategories = false
 
+    this.#actionApiPath = 'w/api.php'
+    this.#restApiPath = 'api/rest_v1'
+    this.#wikiPath = 'wiki/'
+    this.#modulePathOpt = 'w/load.php'
+
     this.namespaces = {}
     this.namespacesToMirror = []
-
-    this.#apiActionPath = 'w/api.php'
-    this.#wikiPath = 'wiki/'
     this.apiCheckArticleId = 'MediaWiki:Sidebar'
 
     this.queryOpts = {
@@ -152,7 +179,7 @@ class MediaWiki {
 
   public async hasVisualEditorApi(): Promise<boolean> {
     if (this.#hasVisualEditorApi === null) {
-      this.#hasVisualEditorApi = await checkApiAvailability(this.VisualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
+      this.#hasVisualEditorApi = await checkApiAvailability(this.visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
       return this.#hasVisualEditorApi
     }
     return this.#hasVisualEditorApi
@@ -178,23 +205,24 @@ class MediaWiki {
   }
 
   private initMWApis() {
-    const baseUrlDirector = new BaseURLDirector(this.baseUrl.href)
-    this.webUrl = baseUrlDirector.buildURL(this.#wikiPath)
-    this.apiUrl = baseUrlDirector.buildURL(this.#apiActionPath)
-    this.apiUrlDirector = new ApiURLDirector(this.apiUrl.href)
-    this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
-    this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl(this.#apiPath)
-    this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl(this.#apiPath)
-    this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt)
-    this.mobileModulePath = baseUrlDirector.buildMobileModuleURL()
+    this.WikimediaDesktopApiUrl = this.baseUrlDirector.buildWikimediaDesktopApiUrl(this.#restApiPath)
+    this.WikimediaMobileApiUrl = this.baseUrlDirector.buildWikimediaMobileApiUrl(this.#restApiPath)
+    this.mobileModulePath = this.baseUrlDirector.buildMobileModuleURL()
     this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href)
     this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href)
-    this.VisualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
+  }
+
+  private initApiURLDirector() {
+    this.webUrl = this.baseUrlDirector.buildURL(this.#wikiPath)
+    this.actionApiUrl = this.baseUrlDirector.buildURL(this.#actionApiPath)
+    this.apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href)
+    this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
+    this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
   }
 
   public async login(downloader: Downloader) {
     if (this.#username && this.#password) {
-      let url = this.apiUrl.href + '?'
+      let url = this.actionApiUrl.href + '?'
 
       // Add domain if configured
       if (this.#domain) {
@@ -205,7 +233,7 @@ class MediaWiki {
       const { content, responseHeaders } = await downloader.downloadContent(url + 'action=query&meta=tokens&type=login&format=json&formatversion=2')
 
       // Logging in
-      await axios(this.apiUrl.href, {
+      await axios(this.actionApiUrl.href, {
         data: qs.stringify({
           action: 'login',
           format: 'json',
@@ -413,13 +441,15 @@ class MediaWiki {
 
     const mwMetaData: MWMetaData = {
       webUrl: this.webUrl.href,
-      apiUrl: this.apiUrl.href,
+      actionApiUrl: this.actionApiUrl.href,
+      modulePathOpt: this.#modulePathOpt,
       modulePath: this.modulePath,
       mobileModulePath: this.mobileModulePath,
       webUrlPath: this.webUrl.pathname,
       wikiPath: this.#wikiPath,
       baseUrl: this.baseUrl.href,
-      apiActionPath: this.#apiActionPath,
+      actionApiPath: this.#actionApiPath,
+      restApiPath: this.#restApiPath,
       domain: this.#domain,
 
       textDir: textDir as TextDirection,

src/mwoffliner.lib.ts

@@ -76,7 +76,7 @@ async function execute(argv: any) {
     mwUrl,
     mwWikiPath,
     mwActionApiPath,
-    mwApiPath,
+    mwRestApiPath,
     mwModulePath,
     mwDomain,
     mwUsername,
@@ -158,13 +158,13 @@ async function execute(argv: any) {
   /* Wikipedia/... URL; Normalize by adding trailing / as necessary */
   MediaWiki.base = mwUrl
   MediaWiki.getCategories = !!argv.getCategories
-  MediaWiki.apiActionPath = mwActionApiPath
-  MediaWiki.apiPath = mwApiPath
+  MediaWiki.wikiPath = mwWikiPath
+  MediaWiki.actionApiPath = mwActionApiPath
+  MediaWiki.restApiPath = mwRestApiPath
   MediaWiki.modulePathOpt = mwModulePath
   MediaWiki.domain = mwDomain
   MediaWiki.password = mwPassword
   MediaWiki.username = mwUsername
-  MediaWiki.wikiPath = mwWikiPath
 
   /* Download helpers; TODO: Merge with something else / expand this. */
   const downloader = new Downloader({
@@ -513,7 +513,7 @@ async function execute(argv: any) {
       }
     }
 
-    const apiUrlDirector = new ApiURLDirector(MediaWiki.apiUrl.href)
+    const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)
 
     const body = await downloader.getJSON<any>(apiUrlDirector.buildSiteInfoURL())
 

src/parameterList.ts

@@ -17,8 +17,8 @@ export const parameterDescriptions = {
     'Specify a flavour for the scraping. If missing, scrape all article contents. Each --format argument will cause a new local file to be created but options can be combined. Supported options are:\n * novid: no video & audio content\n * nopic: no pictures (implies "novid")\n * nopdf: no PDF files\n * nodet: only the first/head paragraph (implies "novid")\nFormat names can also be aliased using a ":"\nExample: "... --format=nopic:mini --format=novid,nopdf"',
   keepEmptyParagraphs: 'Keep all paragraphs, even empty ones.',
   mwWikiPath: 'Mediawiki wiki base path (per default "/wiki/")',
-  mwActionApiPath: 'Mediawiki action API path (per default "/w/api.php")',
-  mwApiPath: 'Mediawiki Rest API path (per default "/api/rest_v1")',
+  mwActionApiPath: 'Mediawiki API path (per default "/w/api.php")',
+  mwRestApiPath: 'Mediawiki Rest API path (per default "/api/rest_v1")',
   mwModulePath: 'Mediawiki module load path (per default "/w/load.php")',
   mwDomain: 'Mediawiki user domain (thought for private wikis)',
   mwUsername: 'Mediawiki username (thought for private wikis)',

src/renderers/renderer.builder.ts

@@ -47,30 +47,35 @@ export class RendererBuilder {
         }
       case 'specific':
         // renderName argument is required for 'specific' mode
-        switch (renderName) {
-          case 'WikimediaDesktop':
-            if (hasWikimediaDesktopApi) {
-              return new WikimediaDesktopRenderer()
-            }
-            logger.error('Cannot create an instance of WikimediaDesktop renderer.')
-            process.exit(1)
-          case 'VisualEditor':
-            if (hasVisualEditorApi) {
-              return new VisualEditorRenderer()
-            }
-            logger.error('Cannot create an instance of VisualEditor renderer.')
-            process.exit(1)
-          case 'WikimediaMobile':
-            if (hasWikimediaMobileApi) {
-              return new WikimediaMobileRenderer()
-            }
-            logger.error('No available mobile renderer.')
-            process.exit(1)
-          default:
-            throw new Error(`Unknown renderName for specific mode: ${renderName}`)
-        }
+        return this.handleSpecificRender(renderName, hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi)
       default:
         throw new Error(`Unknown render: ${renderType}`)
     }
   }
+
+  private handleSpecificRender(renderName: string, hasVisualEditorApi: boolean, hasWikimediaDesktopApi: boolean, hasWikimediaMobileApi: boolean) {
+    // renderName argument is required for 'specific' mode
+    switch (renderName) {
+      case 'WikimediaDesktop':
+        if (hasWikimediaDesktopApi) {
+          return new WikimediaDesktopRenderer()
+        }
+        logger.error('Cannot create an instance of WikimediaDesktop renderer.')
+        process.exit(1)
+      case 'VisualEditor':
+        if (hasVisualEditorApi) {
+          return new VisualEditorRenderer()
+        }
+        logger.error('Cannot create an instance of VisualEditor renderer.')
+        process.exit(1)
+      case 'WikimediaMobile':
+        if (hasWikimediaMobileApi) {
+          return new WikimediaMobileRenderer()
+        }
+        logger.error('No available mobile renderer.')
+        process.exit(1)
+      default:
+        throw new Error(`Unknown renderName for specific mode: ${renderName}`)
+    }
+  }
 }

src/sanitize-argument.ts

@@ -31,6 +31,10 @@ export async function sanitize_all(argv: any) {
     customZimLongDescription,
     customZimDescription,
     forceRender,
+    mwWikiPath,
+    mwActionApiPath,
+    mwRestApiPath,
+    mwModulePath,
   } = argv
 
   sanitizeDoubleUsedParameters(argv)
@@ -78,6 +82,18 @@ export async function sanitize_all(argv: any) {
     throw err
   })
 
+  // sanitizing mwWikiPath
+  sanitizeWikiPath(mwWikiPath)
+
+  // sanitizing mwRestApiPath
+  sanitizeApiPathParam(mwRestApiPath)
+
+  // sanitizing mwActionApiPath
+  sanitizeApiPathParam(mwActionApiPath)
+
+  // sanitizing mwModulePath
+  sanitizeApiPathParam(mwModulePath)
+
   // sanitize Custom Main Page
   if (argv.customMainPage) {
     argv.customMainPage = argv.customMainPage.replace(/ /g, '_')
@@ -103,6 +119,30 @@ export async function sanitize_all(argv: any) {
   }
 }
 
+export function sanitizeWikiPath(mwWikiPath = '') {
+  mwWikiPath = sanitizeApiPathParam(mwWikiPath)
+
+  // Make sure wikiPath always has forward slash at the end for the correct URL building
+  if (!mwWikiPath?.endsWith('/')) {
+    mwWikiPath += '/'
+  }
+
+  return mwWikiPath
+}
+
+export function sanitizeApiPathParam(apiPathParam: string) {
+  if (!apiPathParam) {
+    return
+  }
+
+  // No api params should start from forward slash
+  if (apiPathParam.startsWith('/')) {
+    apiPathParam = apiPathParam.slice(1)
+  }
+
+  return apiPathParam
+}
+
 export function sanitizeStringMaxLength(text: string, key: string, length: number) {
   if (text && text.length > length) {
     throw new Error(`${key} should be less than ${length} characters.`)