From cce65ee8c117b44d0534688c4b96b3534ef0e8cf Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 25 Jul 2019 12:25:05 -0700 Subject: [PATCH] core: add source-maps gatherer (#9101) --- lighthouse-core/gather/driver.js | 2 +- .../gather/gatherers/source-maps.js | 159 ++++++++++++ lighthouse-core/test/gather/driver-test.js | 67 +---- .../test/gather/gatherers/source-maps-test.js | 241 ++++++++++++++++++ lighthouse-core/test/gather/mock-commands.js | 119 +++++++++ types/artifacts.d.ts | 42 +++ 6 files changed, 563 insertions(+), 67 deletions(-) create mode 100644 lighthouse-core/gather/gatherers/source-maps.js create mode 100644 lighthouse-core/test/gather/gatherers/source-maps-test.js create mode 100644 lighthouse-core/test/gather/mock-commands.js diff --git a/lighthouse-core/gather/driver.js b/lighthouse-core/gather/driver.js index b29fbf298715..492a4945a660 100644 --- a/lighthouse-core/gather/driver.js +++ b/lighthouse-core/gather/driver.js @@ -476,7 +476,7 @@ class Driver { const contextId = options.useIsolation ? await this._getOrCreateIsolatedContextId() : undefined; try { - // `await` is not redunant here because we want to `catch` the async errors + // `await` is not redundant here because we want to `catch` the async errors return await this._evaluateInContext(expression, contextId); } catch (err) { // If we were using isolation and the context disappeared on us, retry one more time. diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js new file mode 100644 index 000000000000..17f7405af768 --- /dev/null +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -0,0 +1,159 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** @typedef {import('../driver.js')} Driver */ + +const Gatherer = require('./gatherer.js'); +const URL = require('../../lib/url-shim.js'); + +/** + * This function fetches source maps; it is careful not to parse the response as JSON, as it will + * just need to be serialized again over the protocol, and source maps can + * be huge. + * + * @param {string} url + * @return {Promise} + */ +/* istanbul ignore next */ +async function fetchSourceMap(url) { + // eslint-disable-next-line no-undef + const response = await fetch(url); + if (response.ok) { + return response.text(); + } else { + throw new Error(`Received status code ${response.status} for ${url}`); + } +} + +/** + * @fileoverview Gets JavaScript source maps. + */ +class SourceMaps extends Gatherer { + constructor() { + super(); + /** @type {LH.Crdp.Debugger.ScriptParsedEvent[]} */ + this._scriptParsedEvents = []; + this.onScriptParsed = this.onScriptParsed.bind(this); + } + + /** + * @param {Driver} driver + * @param {string} sourceMapUrl + * @return {Promise} + */ + async fetchSourceMapInPage(driver, sourceMapUrl) { + driver.setNextProtocolTimeout(1500); + /** @type {string} */ + const sourceMapJson = + await driver.evaluateAsync(`(${fetchSourceMap})(${JSON.stringify(sourceMapUrl)})`); + return JSON.parse(sourceMapJson); + } + + /** + * @param {string} sourceMapURL + * @return {LH.Artifacts.RawSourceMap} + */ + parseSourceMapFromDataUrl(sourceMapURL) { + const buffer = Buffer.from(sourceMapURL.split(',')[1], 'base64'); + return JSON.parse(buffer.toString()); + } + + /** + * @param {LH.Crdp.Debugger.ScriptParsedEvent} event + */ + onScriptParsed(event) { + if (event.sourceMapURL) { + this._scriptParsedEvents.push(event); + } + } + + /** + * @param {LH.Gatherer.PassContext} passContext + */ + async beforePass(passContext) { + const driver = passContext.driver; + driver.on('Debugger.scriptParsed', this.onScriptParsed); + await driver.sendCommand('Debugger.enable'); + } + + /** + * @param {string} url + * @param {string} base + * @return {string|undefined} + */ + _resolveUrl(url, base) { + try { + return new URL(url, base).href; + } catch (e) { + return; + } + } + + /** + * @param {Driver} driver + * @param {LH.Crdp.Debugger.ScriptParsedEvent} event + * @return {Promise} + */ + async _retrieveMapFromScriptParsedEvent(driver, event) { + if (!event.sourceMapURL) { + throw new Error('precondition failed: event.sourceMapURL should exist'); + } + + // `sourceMapURL` is simply the URL found in either a magic comment or an x-sourcemap header. + // It has not been resolved to a base url. + const isSourceMapADataUri = event.sourceMapURL.startsWith('data:'); + const scriptUrl = event.url; + const rawSourceMapUrl = isSourceMapADataUri ? + event.sourceMapURL : + this._resolveUrl(event.sourceMapURL, event.url); + + if (!rawSourceMapUrl) { + return { + scriptUrl, + errorMessage: `Could not resolve map url: ${event.sourceMapURL}`, + }; + } + + // sourceMapUrl isn't included in the the artifact if it was a data URL. + const sourceMapUrl = isSourceMapADataUri ? undefined : rawSourceMapUrl; + + try { + const map = isSourceMapADataUri ? + this.parseSourceMapFromDataUrl(rawSourceMapUrl) : + await this.fetchSourceMapInPage(driver, rawSourceMapUrl); + return { + scriptUrl, + sourceMapUrl, + map, + }; + } catch (err) { + return { + scriptUrl, + sourceMapUrl, + errorMessage: err.toString(), + }; + } + } + + /** + * @param {LH.Gatherer.PassContext} passContext + * @return {Promise} + */ + async afterPass(passContext) { + const driver = passContext.driver; + + driver.off('Debugger.scriptParsed', this.onScriptParsed); + await driver.sendCommand('Debugger.disable'); + + const eventProcessPromises = this._scriptParsedEvents + .map((event) => this._retrieveMapFromScriptParsedEvent(driver, event)); + + return Promise.all(eventProcessPromises); + } +} + +module.exports = SourceMaps; diff --git a/lighthouse-core/test/gather/driver-test.js b/lighthouse-core/test/gather/driver-test.js index 62bfbcb2c1c7..d78ba2c9757a 100644 --- a/lighthouse-core/test/gather/driver-test.js +++ b/lighthouse-core/test/gather/driver-test.js @@ -10,6 +10,7 @@ const Connection = require('../../gather/connections/connection.js'); const Element = require('../../lib/element.js'); const EventEmitter = require('events').EventEmitter; const {protocolGetVersionResponse} = require('./fake-driver.js'); +const {createMockSendCommandFn, createMockOnceFn} = require('./mock-commands.js'); const redirectDevtoolsLog = require('../fixtures/wikipedia-redirect.devtoolslog.json'); @@ -17,72 +18,6 @@ const redirectDevtoolsLog = require('../fixtures/wikipedia-redirect.devtoolslog. jest.useFakeTimers(); -/** - * Creates a jest mock function whose implementation consumes mocked protocol responses matching the - * requested command in the order they were mocked. - * - * It is decorated with two methods: - * - `mockResponse` which pushes protocol message responses for consumption - * - `findInvocation` which asserts that `sendCommand` was invoked with the given command and - * returns the protocol message argument. - */ -function createMockSendCommandFn() { - const mockResponses = []; - const mockFn = jest.fn().mockImplementation(command => { - const indexOfResponse = mockResponses.findIndex(entry => entry.command === command); - if (indexOfResponse === -1) throw new Error(`${command} unimplemented`); - const {response, delay} = mockResponses[indexOfResponse]; - mockResponses.splice(indexOfResponse, 1); - if (delay) return new Promise(resolve => setTimeout(() => resolve(response), delay)); - return Promise.resolve(response); - }); - - mockFn.mockResponse = (command, response, delay) => { - mockResponses.push({command, response, delay}); - return mockFn; - }; - - mockFn.findInvocation = command => { - expect(mockFn).toHaveBeenCalledWith(command, expect.anything()); - return mockFn.mock.calls.find(call => call[0] === command)[1]; - }; - - return mockFn; -} - -/** - * Creates a jest mock function whose implementation invokes `.on`/`.once` listeners after a setTimeout tick. - * Closely mirrors `createMockSendCommandFn`. - * - * It is decorated with two methods: - * - `mockEvent` which pushes protocol event payload for consumption - * - `findListener` which asserts that `on` was invoked with the given event name and - * returns the listener . - */ -function createMockOnceFn() { - const mockEvents = []; - const mockFn = jest.fn().mockImplementation((eventName, listener) => { - const indexOfResponse = mockEvents.findIndex(entry => entry.event === eventName); - if (indexOfResponse === -1) return; - const {response} = mockEvents[indexOfResponse]; - mockEvents.splice(indexOfResponse, 1); - // Wait a tick because real events never fire immediately - setTimeout(() => listener(response), 0); - }); - - mockFn.mockEvent = (event, response) => { - mockEvents.push({event, response}); - return mockFn; - }; - - mockFn.findListener = event => { - expect(mockFn).toHaveBeenCalledWith(event, expect.anything()); - return mockFn.mock.calls.find(call => call[0] === event)[1]; - }; - - return mockFn; -} - /** * Transparently augments the promise with inspectable functions to query its state. * diff --git a/lighthouse-core/test/gather/gatherers/source-maps-test.js b/lighthouse-core/test/gather/gatherers/source-maps-test.js new file mode 100644 index 000000000000..5d809cbb7496 --- /dev/null +++ b/lighthouse-core/test/gather/gatherers/source-maps-test.js @@ -0,0 +1,241 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-env jest */ + +jest.useFakeTimers(); + +const Driver = require('../../../gather/driver.js'); +const Connection = require('../../../gather/connections/connection.js'); +const SourceMaps = require('../../../gather/gatherers/source-maps.js'); +const {createMockSendCommandFn, createMockOnFn} = require('../mock-commands.js'); + +const mapJson = JSON.stringify({ + version: 3, + file: 'out.js', + sourceRoot: '', + sources: ['foo.js', 'bar.js'], + names: ['src', 'maps', 'are', 'fun'], + mappings: 'AAgBC,SAAQ,CAAEA', +}); + +describe('SourceMaps gatherer', () => { + /** + * `scriptParsedEvent` mocks the `sourceMapURL` and `url` seen from the protocol. + * `map` mocks the (JSON) of the source maps that `Runtime.evaluate` returns. + * `resolvedSourceMapUrl` is used to assert that the SourceMaps gatherer is using the expected + * url to fetch the source map. + * `fetchError` mocks an error that happens in the page. Only fetch error message make sense. + * @param {Array<{scriptParsedEvent: LH.Crdp.Debugger.ScriptParsedEvent, map: string, resolvedSourceMapUrl?: string, fetchError: string}>} mapsAndEvents + * @return {Promise} + */ + async function runSourceMaps(mapsAndEvents) { + const onMock = createMockOnFn(); + + const sendCommandMock = createMockSendCommandFn() + .mockResponse('Debugger.enable', {}) + .mockResponse('Debugger.disable', {}); + + for (const {scriptParsedEvent, map, resolvedSourceMapUrl, fetchError} of mapsAndEvents) { + onMock.mockEvent('protocolevent', { + method: 'Debugger.scriptParsed', + params: scriptParsedEvent, + }); + + if (scriptParsedEvent.sourceMapURL.startsWith('data:')) { + // Only the source maps that need to be fetched use the `evaluateAsync` code path. + continue; + } + + if (map && fetchError) { + throw new Error('should only define map or fetchError, not both.'); + } + + sendCommandMock.mockResponse('Runtime.evaluate', ({expression}) => { + // Check that the source map url was resolved correctly. It'll be somewhere + // in the code sent to Runtime.evaluate. + if (resolvedSourceMapUrl && !expression.includes(resolvedSourceMapUrl)) { + throw new Error(`did not request expected url: ${resolvedSourceMapUrl}`); + } + + const value = fetchError ? + Object.assign(new Error(), {message: fetchError, __failedInBrowser: true}) : + map; + return {result: {value}}; + }); + } + const connectionStub = new Connection(); + connectionStub.sendCommand = sendCommandMock; + connectionStub.on = onMock; + + const driver = new Driver(connectionStub); + + const sourceMaps = new SourceMaps(); + await sourceMaps.beforePass({driver}); + jest.advanceTimersByTime(1); + return sourceMaps.afterPass({driver}); + } + + function makeJsonDataUrl(data) { + return 'data:application/json;charset=utf-8;base64,' + Buffer.from(data).toString('base64'); + } + + it('ignores script with no source map url', async () => { + const artifact = await runSourceMaps([ + { + scriptParsedEvent: { + url: 'http://www.example.com/script.js', + sourceMapURL: '', + }, + map: null, + }, + ]); + expect(artifact).toEqual([]); + }); + + it('fetches map for script with source map url', async () => { + const mapsAndEvents = [ + { + scriptParsedEvent: { + url: 'http://www.example.com/bundle.js', + sourceMapURL: 'http://www.example.com/bundle.js.map', + }, + map: mapJson, + resolvedSourceMapUrl: 'http://www.example.com/bundle.js.map', + }, + ]; + const artifact = await runSourceMaps(mapsAndEvents); + expect(artifact).toEqual([ + { + scriptUrl: mapsAndEvents[0].scriptParsedEvent.url, + sourceMapUrl: mapsAndEvents[0].scriptParsedEvent.sourceMapURL, + map: JSON.parse(mapsAndEvents[0].map), + }, + ]); + }); + + it('fetches map for script with relative source map url', async () => { + const mapsAndEvents = [ + { + scriptParsedEvent: { + url: 'http://www.example.com/path/bundle.js', + sourceMapURL: 'bundle.js.map', + }, + map: mapJson, + resolvedSourceMapUrl: 'http://www.example.com/path/bundle.js.map', + }, + { + scriptParsedEvent: { + url: 'http://www.example.com/path/bundle.js', + sourceMapURL: '../bundle.js.map', + }, + map: mapJson, + resolvedSourceMapUrl: 'http://www.example.com/bundle.js.map', + }, + { + scriptParsedEvent: { + url: 'http://www.example.com/path/bundle.js', + sourceMapURL: 'http://www.example-2.com/path/bundle.js', + }, + map: mapJson, + resolvedSourceMapUrl: 'http://www.example-2.com/path/bundle.js', + }, + ]; + const artifacts = await runSourceMaps(mapsAndEvents); + expect(artifacts).toEqual([ + { + scriptUrl: mapsAndEvents[0].scriptParsedEvent.url, + sourceMapUrl: 'http://www.example.com/path/bundle.js.map', + map: JSON.parse(mapsAndEvents[0].map), + }, + { + scriptUrl: mapsAndEvents[1].scriptParsedEvent.url, + sourceMapUrl: 'http://www.example.com/bundle.js.map', + map: JSON.parse(mapsAndEvents[1].map), + }, + { + scriptUrl: mapsAndEvents[2].scriptParsedEvent.url, + sourceMapUrl: mapsAndEvents[2].scriptParsedEvent.sourceMapURL, + map: JSON.parse(mapsAndEvents[2].map), + }, + ]); + }); + + it('generates an error message when fetching map fails', async () => { + const mapsAndEvents = [ + { + scriptParsedEvent: { + url: 'http://www.example.com/bundle.js', + sourceMapURL: 'http://www.example.com/bundle.js.map', + }, + fetchError: 'Failed fetching source map', + }, + ]; + const artifact = await runSourceMaps(mapsAndEvents); + expect(artifact).toEqual([ + { + scriptUrl: mapsAndEvents[0].scriptParsedEvent.url, + sourceMapUrl: mapsAndEvents[0].scriptParsedEvent.sourceMapURL, + errorMessage: 'Error: Failed fetching source map', + map: undefined, + }, + ]); + }); + + it('generates an error message when map url cannot be resolved', async () => { + const mapsAndEvents = [ + { + scriptParsedEvent: { + url: 'http://www.example.com/bundle.js', + sourceMapURL: 'http://', + }, + }, + ]; + const artifact = await runSourceMaps(mapsAndEvents); + expect(artifact).toEqual([ + { + scriptUrl: mapsAndEvents[0].scriptParsedEvent.url, + sourceMapUrl: undefined, + errorMessage: 'Could not resolve map url: http://', + map: undefined, + }, + ]); + }); + + it('generates an error message when parsing map fails', async () => { + const mapsAndEvents = [ + { + scriptParsedEvent: { + url: 'http://www.example.com/bundle.js', + sourceMapURL: 'http://www.example.com/bundle.js.map', + }, + map: '{{}', + }, + { + scriptParsedEvent: { + url: 'http://www.example.com/bundle-2.js', + sourceMapURL: makeJsonDataUrl('{};'), + }, + }, + ]; + const artifact = await runSourceMaps(mapsAndEvents); + expect(artifact).toEqual([ + { + scriptUrl: mapsAndEvents[0].scriptParsedEvent.url, + sourceMapUrl: mapsAndEvents[0].scriptParsedEvent.sourceMapURL, + errorMessage: 'SyntaxError: Unexpected token { in JSON at position 1', + map: undefined, + }, + { + scriptUrl: mapsAndEvents[1].scriptParsedEvent.url, + sourceMapUrl: undefined, + errorMessage: 'SyntaxError: Unexpected token ; in JSON at position 2', + map: undefined, + }, + ]); + }); +}); diff --git a/lighthouse-core/test/gather/mock-commands.js b/lighthouse-core/test/gather/mock-commands.js new file mode 100644 index 000000000000..32d7872634cb --- /dev/null +++ b/lighthouse-core/test/gather/mock-commands.js @@ -0,0 +1,119 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** + * @fileoverview Refer to driver-test.js and source-maps-test.js for intended usage. + */ + +/* eslint-env jest */ + +/** + * Creates a jest mock function whose implementation consumes mocked protocol responses matching the + * requested command in the order they were mocked. + * + * It is decorated with two methods: + * - `mockResponse` which pushes protocol message responses for consumption + * - `findInvocation` which asserts that `sendCommand` was invoked with the given command and + * returns the protocol message argument. + */ +function createMockSendCommandFn() { + const mockResponses = []; + const mockFn = jest.fn().mockImplementation((command, ...args) => { + const indexOfResponse = mockResponses.findIndex(entry => entry.command === command); + if (indexOfResponse === -1) throw new Error(`${command} unimplemented`); + const {response, delay} = mockResponses[indexOfResponse]; + mockResponses.splice(indexOfResponse, 1); + const returnValue = typeof response === 'function' ? response(...args) : response; + if (delay) return new Promise(resolve => setTimeout(() => resolve(returnValue), delay)); + return Promise.resolve(returnValue); + }); + + mockFn.mockResponse = (command, response, delay) => { + mockResponses.push({command, response, delay}); + return mockFn; + }; + + mockFn.findInvocation = command => { + expect(mockFn).toHaveBeenCalledWith(command, expect.anything()); + return mockFn.mock.calls.find(call => call[0] === command)[1]; + }; + + return mockFn; +} + +/** + * Creates a jest mock function whose implementation invokes `.on`/`.once` listeners after a setTimeout tick. + * Closely mirrors `createMockSendCommandFn`. + * + * It is decorated with two methods: + * - `mockEvent` which pushes protocol event payload for consumption + * - `findListener` which asserts that `on` was invoked with the given event name and + * returns the listener . + */ +function createMockOnceFn() { + const mockEvents = []; + const mockFn = jest.fn().mockImplementation((eventName, listener) => { + const indexOfResponse = mockEvents.findIndex(entry => entry.event === eventName); + if (indexOfResponse === -1) return; + const {response} = mockEvents[indexOfResponse]; + mockEvents.splice(indexOfResponse, 1); + // Wait a tick because real events never fire immediately + setTimeout(() => listener(response), 0); + }); + + mockFn.mockEvent = (event, response) => { + mockEvents.push({event, response}); + return mockFn; + }; + + mockFn.findListener = event => { + expect(mockFn).toHaveBeenCalledWith(event, expect.anything()); + return mockFn.mock.calls.find(call => call[0] === event)[1]; + }; + + return mockFn; +} + +/** + * Very much like `createMockOnceFn`, but will fire all the events (not just one for every call). + * So it's good for .on w/ many events. + */ +function createMockOnFn() { + const mockEvents = []; + const mockFn = jest.fn().mockImplementation((eventName, listener) => { + const events = mockEvents.filter(entry => entry.event === eventName); + if (!events.length) return; + for (const event of events) { + const indexOfEvent = mockEvents.indexOf(event); + mockEvents.splice(indexOfEvent, 1); + } + // Wait a tick because real events never fire immediately + setTimeout(() => { + for (const event of events) { + listener(event.response); + } + }, 0); + }); + + mockFn.mockEvent = (event, response) => { + mockEvents.push({event, response}); + return mockFn; + }; + + mockFn.findListener = event => { + expect(mockFn).toHaveBeenCalledWith(event, expect.anything()); + return mockFn.mock.calls.find(call => call[0] === event)[1]; + }; + + return mockFn; +} + +module.exports = { + createMockSendCommandFn, + createMockOnceFn, + createMockOnFn, +}; diff --git a/types/artifacts.d.ts b/types/artifacts.d.ts index 8dda838f1517..9e93814da045 100644 --- a/types/artifacts.d.ts +++ b/types/artifacts.d.ts @@ -123,6 +123,8 @@ declare global { RobotsTxt: {status: number|null, content: string|null}; /** Version information for all ServiceWorkers active after the first page load. */ ServiceWorker: {versions: Crdp.ServiceWorker.ServiceWorkerVersion[], registrations: Crdp.ServiceWorker.ServiceWorkerRegistration[]}; + /** Source maps of scripts executed in the page. */ + SourceMaps: Array; /** The status of an offline fetch of the page's start_url. -1 and a explanation if missing or there was an error. */ StartUrl: {statusCode: number, explanation?: string}; /** Information on