Add mobile UI regression test suite (#15)

Add mobile UI regression test suite and generic group param

e.g.
./pixel.js reference --group mobile
./pixel.js reference --group desktop

A new set of tests is added for the mobile group with
- basic test for anon
- basic test for logged in
- test for sidebar

Co-authored-by: Nicholas Ray <[email protected]>

README.md

@@ -60,6 +60,8 @@ Or if you want the reference to be a certain release branch:
 ./pixel.js reference -b origin/wmf/1.37.0-wmf.19
 ```
 
+If you want to run the mobile visual regression test suite pass the `--group mobile` flag.
+
 ### 2) Take test screenshots with changed code
 
 If you want to pull a change or multiple changes down from gerrit, take screenshots with these changes on top of master and then compare these screenshots against the reference screenshots, then
@@ -80,6 +82,8 @@ manually open the file at `report/index.html`.
 Additionally, Pixel runs a server at `http://localhost:3000` (default) which can
 be used to interact with/debug the same server that the tests use.
 
+If you want to run the mobile visual regression test suite pass the `--group mobile` flag.
+
 ### Stopping the services
 
 If you want to stop all of Pixel's services, run:
@@ -119,6 +123,8 @@ All tests are located in [config.js](config.js) and follow
 BackstopJS conventions. For more info on how to change or add tests, please
 refer to the [BackstopJS](https://github.com/garris/BackstopJS) README.
 
+Scenarios for mobile site are defined in configMobile.js.
+
 ### Configuring MediaWiki
 
 All mediawiki config is in [LocalSettings.php](LocalSettings.php) and can be

config.js

@@ -110,7 +110,7 @@ module.exports = {
 		// eslint-disable-next-line camelcase
 		engine_scripts: 'src/engine-scripts',
 		// eslint-disable-next-line camelcase
-		html_report: 'report',
+		html_report: 'report/desktop',
 		// eslint-disable-next-line camelcase
 		ci_report: 'report/ci-report',
 		// eslint-disable-next-line camelcase

configMobile.js

@@ -0,0 +1,45 @@
+const config = require( './config.js' );
+
+const BASE_URL = process.env.MW_SERVER;
+const tests = [
+	{
+		label: 'Tree (#minerva #mobile)',
+		path: '/wiki/Tree?useskin=minerva&useformat=mobile'
+	},
+	{
+		label: 'Test (#minerva #mobile)',
+		path: '/wiki/Test?useskin=minerva&useformat=mobile'
+	},
+	{
+		label: 'Test (#minerva #mobile #logged-in)',
+		path: '/wiki/Test?useskin=minerva&useformat=mobile'
+	},
+	{
+		label: 'Test (#minerva #mobile #mainmenu-open)',
+		path: '/wiki/Test?useskin=minerva&useformat=mobile'
+	},
+	{
+		label: 'Test (#minerva #mobile #logged-in #mainmenu-open)',
+		path: '/wiki/Test?useskin=minerva&useformat=mobile'
+	}
+];
+
+const scenarios = tests.map( ( test ) => {
+	return Object.assign( {
+		selectors: [ 'viewport' ]
+	}, test, {
+		url: `${BASE_URL}${test.path}`
+	} );
+} );
+
+module.exports = Object.assign( {}, config, {
+	scenarios,
+	paths: Object.assign( {}, config.paths, {
+		// eslint-disable-next-line camelcase
+		bitmaps_reference: 'report/reference-screenshots-mobile',
+		// eslint-disable-next-line camelcase
+		bitmaps_test: 'report/test-screenshots',
+		// eslint-disable-next-line camelcase
+		html_report: 'report/mobile'
+	} )
+} );

docker-compose.yml

@@ -55,5 +55,6 @@ services:
     volumes:
       - ./context.json:/pixel/context.json
       - ./config.js:/pixel/config.js
+      - ./configMobile.js:/pixel/configMobile.js
       - ./src:/pixel/src
       - ./report:/pixel/report

pixel.js

@@ -6,7 +6,6 @@ const MAIN_BRANCH = 'master';
 const BatchSpawn = require( './src/BatchSpawn' );
 const batchSpawn = new BatchSpawn( 1 );
 const fs = require( 'fs' );
-const REPORT_PATH = 'report/index.html';
 const CONTEXT_PATH = `${__dirname}/context.json`;
 
 let context;
@@ -29,9 +28,11 @@ async function getLatestReleaseBranch() {
 
 /**
  * @param {'test'|'reference'} type
+ * @param {'mobile'|'desktop'} group
  * @return {Promise<undefined>}
  */
-async function openReportIfNecessary( type ) {
+async function openReportIfNecessary( type, group ) {
+	const REPORT_PATH = `report/${group}/index.html`;
 	const filePathFull = `${__dirname}/${REPORT_PATH}`;
 	const markerString = '<div id="root">';
 	try {
@@ -44,6 +45,7 @@ async function openReportIfNecessary( type ) {
 margin-bottom: 16px;border: 1px solid; padding: 12px 24px;
 word-wrap: break-word; overflow-wrap: break-word; overflow: hidden;
 background-color: #eaecf0; border-color: #a2a9b1;">
+<h2>Test group: <strong>${context.group}</strong></h2>
 <p>Comparing ${context.reference} against ${context.test}.</p>
 <p>Test ran on ${new Date()}</p>
 </div>
@@ -56,6 +58,22 @@ ${markerString}`
 	}
 }
 
+/**
+ * @param {string} groupName
+ * @return {string} path to configuration file.
+ * @throws {Error} for unknown group
+ */
+const getGroupConfig = ( groupName ) => {
+	switch ( groupName ) {
+		case 'desktop':
+			return 'config.js';
+		case 'mobile':
+			return 'configMobile.js';
+		default:
+			throw new Error( `Unknown test group: ${groupName}` );
+	}
+};
+
 /**
  * @param {'test'|'reference'} type
  * @param {any} opts
@@ -69,9 +87,12 @@ async function processCommand( type, opts ) {
 
 			console.log( `Using latest branch "${opts.branch}"` );
 		}
+		const group = opts.group;
 		context[ type ] = opts.branch;
+		context.group = group;
 		// store details of this run.
 		fs.writeFileSync( `${__dirname}/context.json`, JSON.stringify( context ) );
+		const configFile = getGroupConfig( group );
 
 		// Start docker containers.
 		await batchSpawn.spawn( 'docker-compose', [ 'up', '-d' ] );
@@ -83,17 +104,17 @@ async function processCommand( type, opts ) {
 		// Execute Visual regression tests.
 		await batchSpawn.spawn(
 			'docker-compose',
-			[ 'run', '--rm', 'visual-regression', type, '--config', 'config.js' ]
+			[ 'run', '--rm', 'visual-regression', type, '--config', configFile ]
 		).then( async () => {
-			await openReportIfNecessary( type );
+			await openReportIfNecessary( type, group );
 		}, async ( /** @type {Error} */ err ) => {
 			if ( err.message.includes( '130' ) ) {
 				// If user ends subprocess with a sigint, exit early.
 				return;
 			}
 
 			if ( err.message.includes( 'Exit with error code 1' ) ) {
-				return openReportIfNecessary( type );
+				return openReportIfNecessary( type, group );
 			}
 
 			throw err;
@@ -116,6 +137,11 @@ function setupCli() {
 		'-c, --change-id <Change-Id...>',
 		'The Change-Id to use. Use multiple flags to use multiple Change-Ids (e.g. -c <Change-Id> -c <Change-Id>)'
 	] );
+	const groupOpt = /** @type {const} */ ( [
+		'-g, --group <(mobile|desktop)>',
+		'The group of tests to run. If omitted the group will be desktop.',
+		'desktop'
+	] );
 
 	program
 		.name( 'pixel.js' )
@@ -126,6 +152,7 @@ function setupCli() {
 		.description( 'Create reference (baseline) screenshots and delete the old reference screenshots.' )
 		.requiredOption( ...branchOpt )
 		.option( ...changeIdOpt )
+		.option( ...groupOpt )
 		.action( ( opts ) => {
 			processCommand( 'reference', opts );
 		} );
@@ -135,6 +162,7 @@ function setupCli() {
 		.description( 'Create test screenshots and compare them against the reference screenshots.' )
 		.requiredOption( ...branchOpt )
 		.option( ...changeIdOpt )
+		.option( ...groupOpt )
 		.action( ( opts ) => {
 			processCommand( 'test', opts );
 		} );

src/engine-scripts/puppet/minerva/mainMenuState.js

@@ -0,0 +1,18 @@
+const menuState = require( '../menuState' );
+
+/**
+ * Open or close Vector-2022's sidebar.
+ *
+ * @param {import('puppeteer').Page} page
+ * @param {string[]} hashtags
+ */
+module.exports = async ( page, hashtags ) => {
+	const isOpen = hashtags.includes( '#mainmenu-open' );
+	const isClosed = hashtags.includes( '#mainmenu-closed' );
+
+	if ( !isOpen && !isClosed ) {
+		return;
+	}
+
+	await menuState( page, '#mw-mf-main-menu-button', isClosed );
+};

src/engine-scripts/puppet/onReady.js

@@ -19,6 +19,11 @@ module.exports = async ( page, scenario ) => {
 		await require( './userMenuState' )( page, hashtags );
 	}
 
+	// These only apply to Minerva
+	if ( hashtags.includes( '#minerva' ) ) {
+		await require( './minerva/mainMenuState' )( page, hashtags );
+	}
+
 	// Make sure the main skin JavaScript module has loaded.
 	await require( './jsReady' )( page );
 	// add more ready handlers here...