core: add source-maps gatherer (#9101)

lighthouse-core/gather/driver.js

@@ -476,7 +476,7 @@ class Driver {
     const contextId = options.useIsolation ? await this._getOrCreateIsolatedContextId() : undefined;
 
     try {
-      // `await` is not redunant here because we want to `catch` the async errors
+      // `await` is not redundant here because we want to `catch` the async errors
       return await this._evaluateInContext(expression, contextId);
     } catch (err) {
       // If we were using isolation and the context disappeared on us, retry one more time.

lighthouse-core/gather/gatherers/source-maps.js

@@ -0,0 +1,159 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/** @typedef {import('../driver.js')} Driver */
+
+const Gatherer = require('./gatherer.js');
+const URL = require('../../lib/url-shim.js');
+
+/**
+ * This function fetches source maps; it is careful not to parse the response as JSON, as it will
+ * just need to be serialized again over the protocol, and source maps can
+ * be huge.
+ *
+ * @param {string} url
+ * @return {Promise<string>}
+ */
+/* istanbul ignore next */
+async function fetchSourceMap(url) {
+  // eslint-disable-next-line no-undef
+  const response = await fetch(url);
+  if (response.ok) {
+    return response.text();
+  } else {
+    throw new Error(`Received status code ${response.status} for ${url}`);
+  }
+}
+
+/**
+ * @fileoverview Gets JavaScript source maps.
+ */
+class SourceMaps extends Gatherer {
+  constructor() {
+    super();
+    /** @type {LH.Crdp.Debugger.ScriptParsedEvent[]} */
+    this._scriptParsedEvents = [];
+    this.onScriptParsed = this.onScriptParsed.bind(this);
+  }
+
+  /**
+   * @param {Driver} driver
+   * @param {string} sourceMapUrl
+   * @return {Promise<LH.Artifacts.RawSourceMap>}
+   */
+  async fetchSourceMapInPage(driver, sourceMapUrl) {
+    driver.setNextProtocolTimeout(1500);
+    /** @type {string} */
+    const sourceMapJson =
+      await driver.evaluateAsync(`(${fetchSourceMap})(${JSON.stringify(sourceMapUrl)})`);
+    return JSON.parse(sourceMapJson);
+  }
+
+  /**
+   * @param {string} sourceMapURL
+   * @return {LH.Artifacts.RawSourceMap}
+   */
+  parseSourceMapFromDataUrl(sourceMapURL) {
+    const buffer = Buffer.from(sourceMapURL.split(',')[1], 'base64');
+    return JSON.parse(buffer.toString());
+  }
+
+  /**
+   * @param {LH.Crdp.Debugger.ScriptParsedEvent} event
+   */
+  onScriptParsed(event) {
+    if (event.sourceMapURL) {
+      this._scriptParsedEvents.push(event);
+    }
+  }
+
+  /**
+   * @param {LH.Gatherer.PassContext} passContext
+   */
+  async beforePass(passContext) {
+    const driver = passContext.driver;
+    driver.on('Debugger.scriptParsed', this.onScriptParsed);
+    await driver.sendCommand('Debugger.enable');
+  }
+
+  /**
+   * @param {string} url
+   * @param {string} base
+   * @return {string|undefined}
+   */
+  _resolveUrl(url, base) {
+    try {
+      return new URL(url, base).href;
+    } catch (e) {
+      return;
+    }
+  }
+
+  /**
+   * @param {Driver} driver
+   * @param {LH.Crdp.Debugger.ScriptParsedEvent} event
+   * @return {Promise<LH.Artifacts.SourceMap>}
+   */
+  async _retrieveMapFromScriptParsedEvent(driver, event) {
+    if (!event.sourceMapURL) {
+      throw new Error('precondition failed: event.sourceMapURL should exist');
+    }
+
+    // `sourceMapURL` is simply the URL found in either a magic comment or an x-sourcemap header.
+    // It has not been resolved to a base url.
+    const isSourceMapADataUri = event.sourceMapURL.startsWith('data:');
+    const scriptUrl = event.url;
+    const rawSourceMapUrl = isSourceMapADataUri ?
+        event.sourceMapURL :
+        this._resolveUrl(event.sourceMapURL, event.url);
+
+    if (!rawSourceMapUrl) {
+      return {
+        scriptUrl,
+        errorMessage: `Could not resolve map url: ${event.sourceMapURL}`,
+      };
+    }
+
+    // sourceMapUrl isn't included in the the artifact if it was a data URL.
+    const sourceMapUrl = isSourceMapADataUri ? undefined : rawSourceMapUrl;
+
+    try {
+      const map = isSourceMapADataUri ?
+          this.parseSourceMapFromDataUrl(rawSourceMapUrl) :
+          await this.fetchSourceMapInPage(driver, rawSourceMapUrl);
+      return {
+        scriptUrl,
+        sourceMapUrl,
+        map,
+      };
+    } catch (err) {
+      return {
+        scriptUrl,
+        sourceMapUrl,
+        errorMessage: err.toString(),
+      };
+    }
+  }
+
+  /**
+   * @param {LH.Gatherer.PassContext} passContext
+   * @return {Promise<LH.Artifacts['SourceMaps']>}
+   */
+  async afterPass(passContext) {
+    const driver = passContext.driver;
+
+    driver.off('Debugger.scriptParsed', this.onScriptParsed);
+    await driver.sendCommand('Debugger.disable');
+
+    const eventProcessPromises = this._scriptParsedEvents
+      .map((event) => this._retrieveMapFromScriptParsedEvent(driver, event));
+
+    return Promise.all(eventProcessPromises);
+  }
+}
+
+module.exports = SourceMaps;

lighthouse-core/test/gather/driver-test.js

@@ -10,79 +10,14 @@ const Connection = require('../../gather/connections/connection.js');
 const Element = require('../../lib/element.js');
 const EventEmitter = require('events').EventEmitter;
 const {protocolGetVersionResponse} = require('./fake-driver.js');
+const {createMockSendCommandFn, createMockOnceFn} = require('./mock-commands.js');
 
 const redirectDevtoolsLog = require('../fixtures/wikipedia-redirect.devtoolslog.json');
 
 /* eslint-env jest */
 
 jest.useFakeTimers();
 
-/**
- * Creates a jest mock function whose implementation consumes mocked protocol responses matching the
- * requested command in the order they were mocked.
- *
- * It is decorated with two methods:
- *    - `mockResponse` which pushes protocol message responses for consumption
- *    - `findInvocation` which asserts that `sendCommand` was invoked with the given command and
- *      returns the protocol message argument.
- */
-function createMockSendCommandFn() {
-  const mockResponses = [];
-  const mockFn = jest.fn().mockImplementation(command => {
-    const indexOfResponse = mockResponses.findIndex(entry => entry.command === command);
-    if (indexOfResponse === -1) throw new Error(`${command} unimplemented`);
-    const {response, delay} = mockResponses[indexOfResponse];
-    mockResponses.splice(indexOfResponse, 1);
-    if (delay) return new Promise(resolve => setTimeout(() => resolve(response), delay));
-    return Promise.resolve(response);
-  });
-
-  mockFn.mockResponse = (command, response, delay) => {
-    mockResponses.push({command, response, delay});
-    return mockFn;
-  };
-
-  mockFn.findInvocation = command => {
-    expect(mockFn).toHaveBeenCalledWith(command, expect.anything());
-    return mockFn.mock.calls.find(call => call[0] === command)[1];
-  };
-
-  return mockFn;
-}
-
-/**
- * Creates a jest mock function whose implementation invokes `.on`/`.once` listeners after a setTimeout tick.
- * Closely mirrors `createMockSendCommandFn`.
- *
- * It is decorated with two methods:
- *    - `mockEvent` which pushes protocol event payload for consumption
- *    - `findListener` which asserts that `on` was invoked with the given event name and
- *      returns the listener .
- */
-function createMockOnceFn() {
-  const mockEvents = [];
-  const mockFn = jest.fn().mockImplementation((eventName, listener) => {
-    const indexOfResponse = mockEvents.findIndex(entry => entry.event === eventName);
-    if (indexOfResponse === -1) return;
-    const {response} = mockEvents[indexOfResponse];
-    mockEvents.splice(indexOfResponse, 1);
-    // Wait a tick because real events never fire immediately
-    setTimeout(() => listener(response), 0);
-  });
-
-  mockFn.mockEvent = (event, response) => {
-    mockEvents.push({event, response});
-    return mockFn;
-  };
-
-  mockFn.findListener = event => {
-    expect(mockFn).toHaveBeenCalledWith(event, expect.anything());
-    return mockFn.mock.calls.find(call => call[0] === event)[1];
-  };
-
-  return mockFn;
-}
-
 /**
  * Transparently augments the promise with inspectable functions to query its state.
  *