core(i18n): add locale fallback when language not supported (#5746)

lighthouse-core/config/config.js

@@ -8,7 +8,8 @@
 const defaultConfigPath = './default-config.js';
 const defaultConfig = require('./default-config.js');
 const fullConfig = require('./full-config.js');
-const constants = require('./constants');
+const constants = require('./constants.js');
+const i18n = require('./../lib/i18n.js');
 
 const isDeepEqual = require('lodash.isequal');
 const log = require('lighthouse-logger');
@@ -401,18 +402,26 @@ class Config {
   }
 
   /**
-   * @param {LH.Config.SettingsJson=} settings
+   * @param {LH.Config.SettingsJson=} settingsJson
    * @param {LH.Flags=} flags
    * @return {LH.Config.Settings}
    */
-  static initSettings(settings = {}, flags) {
+  static initSettings(settingsJson = {}, flags) {
+    // If a locale is requested in flags or settings, use it. A typical CLI run will not have one,
+    // however `lookupLocale` will always determine which of our supported locales to use (falling
+    // back if necessary).
+    const locale = i18n.lookupLocale((flags && flags.locale) || settingsJson.locale);
+
     // Fill in missing settings with defaults
     const {defaultSettings} = constants;
-    const settingWithDefaults = merge(deepClone(defaultSettings), settings, true);
+    const settingWithDefaults = merge(deepClone(defaultSettings), settingsJson, true);
 
     // Override any applicable settings with CLI flags
     const settingsWithFlags = merge(settingWithDefaults || {}, cleanFlagsForSettings(flags), true);
 
+    // Locale is special and comes only from flags/settings/lookupLocale.
+    settingsWithFlags.locale = locale;
+
     return settingsWithFlags;
   }
 

lighthouse-core/config/constants.js

@@ -39,7 +39,7 @@ const defaultSettings = {
 
   // the following settings have no defaults but we still want ensure that `key in settings`
   // in config will work in a typechecked way
-  locale: null, // default determined by the intl library
+  locale: 'en-US', // actual default determined by Config using lib/i18n
   blockedUrlPatterns: null,
   additionalTraceCategories: null,
   extraHeaders: null,

lighthouse-core/lib/i18n.js

@@ -10,6 +10,7 @@ const isDeepEqual = require('lodash.isequal');
 const log = require('lighthouse-logger');
 const MessageFormat = require('intl-messageformat').default;
 const MessageParser = require('intl-messageformat-parser');
+const lookupClosestLocale = require('lookup-closest-locale');
 const LOCALES = require('./locales');
 
 const LH_ROOT = path.join(__dirname, '../../');
@@ -66,6 +67,24 @@ const formats = {
   },
 };
 
+/**
+ * Look up the best available locale for the requested language through these fall backs:
+ * - exact match
+ * - progressively shorter prefixes (`de-CH-1996` -> `de-CH` -> `de`)
+ * - the default locale ('en-US') if no match is found
+ *
+ * If `locale` isn't provided, the default is used.
+ * @param {string=} locale
+ * @return {LH.Locale}
+ */
+function lookupLocale(locale) {
+  // TODO: could do more work to sniff out default locale
+  const canonicalLocale = Intl.getCanonicalLocales(locale)[0];
+
+  const closestLocale = lookupClosestLocale(canonicalLocale, LOCALES);
+  return closestLocale || 'en-US';
+}
+
 /**
  * @param {string} icuMessage
  * @param {Record<string, *>} [values]
@@ -118,7 +137,7 @@ const _icuMessageInstanceMap = new Map();
  * @return {{formattedString: string, icuMessage: string}}
  */
 function _formatIcuMessage(locale, icuMessageId, icuMessage, values) {
-  const localeMessages = LOCALES[locale] || {};
+  const localeMessages = LOCALES[locale];
   const localeMessage = localeMessages[icuMessageId] && localeMessages[icuMessageId].message;
   // fallback to the original english message if we couldn't find a message in the specified locale
   // better to have an english message than no message at all, in some number cases it won't even matter
@@ -150,15 +169,6 @@ function _formatPathAsString(pathInLHR) {
   return pathAsString;
 }
 
-/**
- * @return {LH.Locale}
- */
-function getDefaultLocale() {
-  const defaultLocale = MessageFormat.defaultLocale;
-  if (defaultLocale in LOCALES) return /** @type {LH.Locale} */ (defaultLocale);
-  return 'en-US';
-}
-
 /**
  * @param {LH.Locale} locale
  * @return {LH.I18NRendererStrings}
@@ -208,7 +218,7 @@ function createMessageInstanceIdFn(filename, fileStrings) {
 
 /**
  * @param {string} icuMessageIdOrRawString
- * @param {LH.Locale} [locale]
+ * @param {LH.Locale} locale
  * @return {string}
  */
 function getFormatted(icuMessageIdOrRawString, locale) {
@@ -221,10 +231,10 @@ function getFormatted(icuMessageIdOrRawString, locale) {
 
 /**
  * @param {string} icuMessageInstanceId
- * @param {LH.Locale} [locale]
+ * @param {LH.Locale} locale
  * @return {{icuMessageInstance: IcuMessageInstance, formattedString: string}}
  */
-function _resolveIcuMessageInstanceId(icuMessageInstanceId, locale = 'en-US') {
+function _resolveIcuMessageInstanceId(icuMessageInstanceId, locale) {
   const matches = icuMessageInstanceId.match(MESSAGE_INSTANCE_ID_REGEX);
   if (!matches) throw new Error(`${icuMessageInstanceId} is not a valid message instance ID`);
 
@@ -282,7 +292,7 @@ function replaceIcuMessageInstanceIds(lhr, locale) {
 module.exports = {
   _formatPathAsString,
   UIStrings,
-  getDefaultLocale,
+  lookupLocale,
   getRendererFormattedStrings,
   createMessageInstanceIdFn,
   getFormatted,

lighthouse-core/lib/locales/index.js

@@ -3,11 +3,17 @@
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
-// @ts-nocheck
 'use strict';
 
-module.exports = {
+/** @typedef {Record<string, {message: string}>} LocaleMessages */
+
+/** @type {Record<LH.Locale, LocaleMessages>} */
+const locales = {
+  'ar': require('./ar-XB.json'), // TODO: fallback not needed when ar translation available
+  'ar-XB': require('./ar-XB.json'),
+  'en': require('./en-US.json'), // en-* fallback
   'en-US': require('./en-US.json'),
   'en-XA': require('./en-XA.json'),
-  'ar-XB': require('./ar-XB.json'),
 };
+
+module.exports = locales;

lighthouse-core/runner.js

@@ -32,7 +32,6 @@ class Runner {
     try {
       const startTime = Date.now();
       const settings = opts.config.settings;
-      settings.locale = settings.locale || i18n.getDefaultLocale();
 
       /**
        * List of top-level warnings for this Lighthouse run.
@@ -218,7 +217,7 @@ class Runner {
    */
   static async _runAudit(auditDefn, artifacts, settings, runWarnings) {
     const audit = auditDefn.implementation;
-    const status = `Evaluating: ${i18n.getFormatted(audit.meta.title)}`;
+    const status = `Evaluating: ${i18n.getFormatted(audit.meta.title, 'en-US')}`;
 
     log.log('status', status);
     let auditResult;

lighthouse-core/test/config/config-test.js

@@ -12,6 +12,7 @@ const defaultConfig = require('../../config/default-config.js');
 const log = require('lighthouse-logger');
 const Gatherer = require('../../gather/gatherers/gatherer');
 const Audit = require('../../audits/audit');
+const i18n = require('../../lib/i18n');
 
 /* eslint-env jest */
 
@@ -597,6 +598,29 @@ describe('Config', () => {
     assert.ok(typeof config.settings.maxWaitForLoad === 'number', 'missing setting from default');
   });
 
+  describe('locale', () => {
+    it('falls back to default locale if none specified', () => {
+      const config = new Config({settings: undefined});
+      // Don't assert specific locale so it isn't tied to where tests are run, but
+      // check that it's valid and available.
+      assert.ok(config.settings.locale);
+      assert.strictEqual(config.settings.locale, i18n.lookupLocale(config.settings.locale));
+    });
+
+    it('uses config setting for locale if set', () => {
+      const locale = 'ar-XB';
+      const config = new Config({settings: {locale}});
+      assert.strictEqual(config.settings.locale, locale);
+    });
+
+    it('uses flag setting for locale if set', () => {
+      const settingsLocale = 'en-XA';
+      const flagsLocale = 'ar-XB';
+      const config = new Config({settings: {locale: settingsLocale}}, {locale: flagsLocale});
+      assert.strictEqual(config.settings.locale, flagsLocale);
+    });
+  });
+
   it('is idempotent when accepting a canonicalized Config as valid ConfigJson input', () => {
     const config = new Config(defaultConfig);
     const configAgain = new Config(config);

lighthouse-core/test/lib/i18n-test.js

@@ -62,4 +62,18 @@ describe('i18n', () => {
       expect(strings.scorescaleLabel).toEqual('[Šçöŕé šçåļé: one two]');
     });
   });
+
+  describe('#lookupLocale', () => {
+    it('canonicalizes the locale', () => {
+      expect(i18n.lookupLocale('en-xa')).toEqual('en-XA');
+    });
+
+    it('falls back to root tag prefix if specific locale not available', () => {
+      expect(i18n.lookupLocale('en-JKJK')).toEqual('en');
+    });
+
+    it('falls back to en-US if no match is available', () => {
+      expect(i18n.lookupLocale('jk-Latn-DE-1996-a-ext-x-phonebk-i-klingon')).toEqual('en-US');
+    });
+  });
 });

lighthouse-core/test/lib/locales/index-test.js

@@ -0,0 +1,27 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const locales = require('../../../lib/locales/index.js');
+const assert = require('assert');
+
+/* eslint-env jest */
+
+describe('locales', () => {
+  it('has only canonical language tags', () => {
+    for (const locale of Object.keys(locales)) {
+      const canonicalLocale = Intl.getCanonicalLocales(locale)[0];
+      assert.strictEqual(locale, canonicalLocale);
+    }
+  });
+
+  it('has a base language prefix fallback for all supported languages', () => {
+    for (const locale of Object.keys(locales)) {
+      const basePrefix = locale.split('-')[0];
+      assert.ok(locales[basePrefix]);
+    }
+  });
+});

lighthouse-extension/app/src/lighthouse-background.js

@@ -8,7 +8,6 @@
 const RawProtocol = require('../../../lighthouse-core/gather/connections/raw');
 const Runner = require('../../../lighthouse-core/runner');
 const Config = require('../../../lighthouse-core/config/config');
-const i18n = require('../../../lighthouse-core/lib/i18n');
 const defaultConfig = require('../../../lighthouse-core/config/default-config.js');
 const log = require('lighthouse-logger');
 
@@ -28,7 +27,6 @@ function runLighthouseForConnection(
   const config = new Config({
     extends: 'lighthouse:default',
     settings: {
-      locale: i18n.getDefaultLocale(),
       onlyCategories: categoryIDs,
     },
   }, options.flags);

package.json

@@ -124,6 +124,7 @@
     "js-library-detector": "^4.3.1",
     "lighthouse-logger": "^1.0.0",
     "lodash.isequal": "^4.5.0",
+    "lookup-closest-locale": "6.0.4",
     "metaviewport-parser": "0.2.0",
     "mkdirp": "0.5.1",
     "opn": "4.0.2",

tsconfig.json

@@ -13,6 +13,7 @@
       "./typings"
     ],
 
+    "resolveJsonModule": true,
     "diagnostics": true
   },
   "include": [

typings/externs.d.ts

@@ -15,6 +15,12 @@ declare global {
     code?: string;
   }
 
+  // Augment Intl to include
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/getCanonicalLocales
+  namespace Intl {
+    var getCanonicalLocales: (locales?: string | Array<string>) => Array<string>;
+  }
+
   /** Make properties K in T optional. */
   type MakeOptional<T, K extends keyof T> = {
     [P in Exclude<keyof T, K>]: T[P]
@@ -52,13 +58,13 @@ declare global {
       cpuSlowdownMultiplier?: number
     }
 
-    export type Locale = 'en-US'|'en-XA'|'ar-XB';
+    export type Locale = 'ar'|'ar-XB'|'en'|'en-US'|'en-XA';
 
     export type OutputMode = 'json' | 'html' | 'csv';
 
     interface SharedFlagsSettings {
       output?: OutputMode|OutputMode[];
-      locale?: Locale | null;
+      locale?: Locale;
       maxWaitForLoad?: number;
       blockedUrlPatterns?: string[] | null;
       additionalTraceCategories?: string | null;

typings/lookup-closest-locale/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+declare module 'lookup-closest-locale' {
+  function lookupClosestLocale(locale: string|undefined, available: Record<LH.Locale, any>): LH.Locale|undefined;
+
+  export = lookupClosestLocale;
+}

yarn.lock

@@ -4168,6 +4168,10 @@ longest@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097"
 
+[email protected]:
+  version "6.0.4"
+  resolved "https://registry.yarnpkg.com/lookup-closest-locale/-/lookup-closest-locale-6.0.4.tgz#1279fed7546a601647bbc980f64423ee990a8590"
+
 loose-envify@^1.0.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.2.0.tgz#69a65aad3de542cf4ee0f4fe74e8e33c709ccb0f"