Backfill the assets removed from minified WordPress bundles (#1604)

## Motivation for the change, related issues

To achieve [offline
support](#1483) we
need to download all WordPress files into the Playground filesystem.
Today Playground downloads WordPress partially on boot and downloads
assets as they are needed using fetch requests.

This PR downloads all assets into the Playground filesystem without
blocking the boot process.

## Implementation details

Zip files are generated using the WordPress build script and are
accessible in the `/wp-VERSION/` folder like any other assets.

The download is triggered after WordPress is installed. The fetch is
async and won't block the boot process.

In case some of the assets are needed immediately on boot they will
still be downloaded using fetch to ensure they are available to
WordPress.

Once the assets are downloaded they are stored in the Playground
filesystem and will be served from there.

## Testing Instructions (or ideally a Blueprint)

- Run `npx nx bundle-wordpress:nightly playground-wordpress-builds`
- Confirm that a
`packages/playground/wordpress-builds/public/wp-nightly/wordpress-static.zip`
file was created
- [Load Playground using this
blueprint](http://127.0.0.1:5400/website-server/?php=8.0&wp=nightly)
- In network tools see that the `wordpress-static.zip` file was
downloaded
- Open `/wp-admin/` 
- In network tools confirm that static assets like _thickbox.js_ were
loaded from the service worker instead of a fetch

---------

Co-authored-by: Adam Zieliński <[email protected]>

packages/playground/common/src/index.ts

@@ -21,7 +21,8 @@ const tmpPath = '/tmp/file.zip';
 export const unzipFile = async (
 	php: UniversalPHP,
 	zipPath: string | File,
-	extractToPath: string
+	extractToPath: string,
+	overwriteFiles = true
 ) => {
 	if (zipPath instanceof File) {
 		const zipFile = zipPath;
@@ -34,25 +35,35 @@ export const unzipFile = async (
 	const js = phpVars({
 		zipPath,
 		extractToPath,
+		overwriteFiles,
 	});
 	await php.run({
 		code: `<?php
-        function unzip($zipPath, $extractTo, $overwrite = true)
+        function unzip($zipPath, $extractTo, $overwriteFiles = true)
         {
             if (!is_dir($extractTo)) {
                 mkdir($extractTo, 0777, true);
             }
             $zip = new ZipArchive;
             $res = $zip->open($zipPath);
             if ($res === TRUE) {
-                $zip->extractTo($extractTo);
-                $zip->close();
-                chmod($extractTo, 0777);
+				for ($i = 0; $i < $zip->numFiles; $i++) {
+					$filename = $zip->getNameIndex($i);
+					$fileinfo = pathinfo($filename);
+					$extractFilePath = rtrim($extractTo, '/') . '/' . $filename;
+					// Check if file exists and $overwriteFiles is false
+					if (!file_exists($extractFilePath) || $overwriteFiles) {
+						// Extract file
+						$zip->extractTo($extractTo, $filename);
+					}
+				}
+				$zip->close();
+				chmod($extractTo, 0777);
             } else {
                 throw new Exception("Could not unzip file");
             }
         }
-        unzip(${js.zipPath}, ${js.extractToPath});
+        unzip(${js.zipPath}, ${js.extractToPath}, ${js.overwriteFiles});
         `,
 	});
 	if (await php.fileExists(tmpPath)) {

packages/playground/remote/src/lib/boot-playground-remote.ts

@@ -205,6 +205,13 @@ export async function bootPlaygroundRemote() {
 		) {
 			return await phpApi.bindOpfs(options, onProgress);
 		},
+
+		/**
+		 * Download WordPress assets.
+		 */
+		async backfillStaticFilesRemovedFromMinifiedBuild() {
+			await webApi.backfillStaticFilesRemovedFromMinifiedBuild();
+		},
 	};
 
 	await phpApi.isConnected();
@@ -236,6 +243,33 @@ export async function bootPlaygroundRemote() {
 		throw e;
 	}
 
+	/**
+	 * When WordPress is loaded from a minified bundle, some assets are removed to reduce the bundle size.
+	 * This function backfills the missing assets. If WordPress is loaded from a non-minified bundle,
+	 * we don't need to backfill because the assets are already included.
+	 *
+	 * If the browser is online we download the WordPress assets asynchronously to speed up the boot process.
+	 * Missing assets will be fetched on demand from the Playground server until they are downloaded.
+	 *
+	 * If the browser is offline, we await the backfill or WordPress assets
+	 * from cache to ensure Playground is fully functional before boot finishes.
+	 */
+	if (window.navigator.onLine) {
+		wpFrame.addEventListener('load', () => {
+			webApi.backfillStaticFilesRemovedFromMinifiedBuild();
+		});
+	} else {
+		// Note this will run even if the static files are already in place, e.g. when running
+		// a non-minified build or an offline site. It doesn't seem like a big problem worth introducing
+		// a new API method like `webApi.needsBackfillingStaticFilesRemovedFromMinifiedBuild().
+		webApi.setProgress({
+			caption: 'Downloading WordPress assets',
+			isIndefinite: false,
+			visible: true,
+		});
+		await webApi.backfillStaticFilesRemovedFromMinifiedBuild();
+	}
+
 	/*
 	 * An assertion to make sure Playground Client is compatible
 	 * with Remote<PlaygroundClient>
@@ -299,7 +333,8 @@ function assertNotInfiniteLoadingLoop() {
 	}
 	if (isBrowserInABrowser) {
 		throw new Error(
-			'The service worker did not load correctly. This is a bug, please report it on https://github.com/WordPress/wordpress-playground/issues'
+			`The service worker did not load correctly. This is a bug,
+			please report it on https://github.com/WordPress/wordpress-playground/issues`
 		);
 	}
 	(window as any).IS_WASM_WORDPRESS = true;

packages/playground/remote/src/lib/playground-client.ts

@@ -52,6 +52,7 @@ export interface WebClientMixin extends ProgressReceiver {
 	replayFSJournal: PlaygroundWorkerEndpoint['replayFSJournal'];
 	addEventListener: PlaygroundWorkerEndpoint['addEventListener'];
 	removeEventListener: PlaygroundWorkerEndpoint['removeEventListener'];
+	backfillStaticFilesRemovedFromMinifiedBuild: PlaygroundWorkerEndpoint['backfillStaticFilesRemovedFromMinifiedBuild'];
 
 	/** @inheritDoc @php-wasm/universal!UniversalPHP.onMessage */
 	onMessage: PlaygroundWorkerEndpoint['onMessage'];

packages/playground/remote/src/lib/worker-thread.ts

@@ -37,13 +37,14 @@ import transportFetch from './playground-mu-plugin/playground-includes/wp_http_f
 import transportDummy from './playground-mu-plugin/playground-includes/wp_http_dummy.php?raw';
 /** @ts-ignore */
 import playgroundWebMuPlugin from './playground-mu-plugin/0-playground.php?raw';
-import { PHPWorker } from '@php-wasm/universal';
+import { PHP, PHPWorker } from '@php-wasm/universal';
 import {
 	bootWordPress,
 	getLoadedWordPressVersion,
 } from '@wp-playground/wordpress';
 import { wpVersionToStaticAssetsDirectory } from '@wp-playground/wordpress-builds';
 import { logger } from '@php-wasm/logger';
+import { unzipFile } from '@wp-playground/common';
 
 /**
  * Startup options are received from spawnPHPWorkerThread using a message event.
@@ -194,6 +195,113 @@ export class PlaygroundWorkerEndpoint extends PHPWorker {
 	async replayFSJournal(events: FilesystemOperation[]) {
 		return replayFSJournal(this.__internal_getPHP()!, events);
 	}
+
+	async backfillStaticFilesRemovedFromMinifiedBuild() {
+		await backfillStaticFilesRemovedFromMinifiedBuild(
+			this.__internal_getPHP()!
+		);
+	}
+}
+
+/**
+ * Downloads and unzips a ZIP bundle of all the static assets removed from
+ * the currently loaded minified WordPress build. Doesn't do anything if the
+ * assets are already downloaded or if a non-minified WordPress build is loaded.
+ *
+ * ## Asset Loading
+ *
+ * To load Playground faster, we default to minified WordPress builds shipped
+ * without most CSS files, JS files, and other static assets.
+ *
+ * When Playground requests a static asset that is not in the minified build, the service
+ * worker consults the list of the assets removed during the minification process. Such
+ * a list is shipped with every minified build in a file called `wordpress-remote-asset-paths`.
+ *
+ * For example, when `/wp-includes/css/dist/block-library/common.min.css` isn't found
+ * in the Playground filesystem, the service worker looks for it in `/wordpress/wordpress-remote-asset-paths`
+ * and finds it there. This means it's available on the remote server, so the service
+ * worker fetches it from an URL like:
+ *
+ * https://playground.wordpress.net/wp-6.5/wp-includes/css/dist/block-library/common.min.css
+ *
+ * ## Assets backfilling
+ *
+ * Running Playground offline isn't possible without shipping all the static assets into the browser.
+ * Downloading every CSS and JS file one request at a time would be slow to run and tedious to maintain.
+ * This is where this function comes in!
+ *
+ * It downloads a zip archive containing all the static files removed from the currently running
+ * minified build, and unzips them in the Playground filesystem. Once it finishes, the WordPress
+ * installation running in the browser is complete and the service worker will no longer have
+ * to backfill any static assets again.
+ *
+ * This process is started after the Playground boots (see `bootPlaygroundRemote`) and the first
+ * page is rendered. This way we're not delaying the initial Playground paint with a large download.
+ *
+ * ## Prevent backfilling if assets are already available
+ *
+ * Running this function twice, or running it on a non-minified build will have no effect.
+ *
+ * The backfilling only runs when a non-empty `wordpress-remote-asset-paths` file
+ * exists. When one is missing, we're not running a minified build. When one is empty,
+ * it means the backfilling process was already done – this function empties the file
+ * after the backfilling is done.
+ *
+ * ### Downloading assets during backfill
+ *
+ * Each WordPress release has a corresponding static assets directory on the Playground.WordPress.net server.
+ * The file is downloaded from the server and unzipped into the WordPress document root.
+ *
+ * ### Skipping existing files during unzipping
+ *
+ * If any of the files already exist, they are skipped and not overwritten.
+ * By skipping existing files, we ensure that the backfill process doesn't overwrite any user changes.
+ */
+async function backfillStaticFilesRemovedFromMinifiedBuild(php: PHP) {
+	if (!php.requestHandler) {
+		logger.warn('No PHP request handler available');
+		return;
+	}
+
+	try {
+		const remoteAssetListPath = joinPaths(
+			php.requestHandler.documentRoot,
+			'wordpress-remote-asset-paths'
+		);
+
+		if (
+			!php.fileExists(remoteAssetListPath) ||
+			(await php.readFileAsText(remoteAssetListPath)) === ''
+		) {
+			return;
+		}
+		const wpVersion = await getLoadedWordPressVersion(php.requestHandler);
+		const staticAssetsDirectory =
+			wpVersionToStaticAssetsDirectory(wpVersion);
+		if (!staticAssetsDirectory) {
+			return;
+		}
+		const response = await fetch(
+			joinPaths('/', staticAssetsDirectory, 'wordpress-static.zip')
+		);
+
+		if (!response.ok) {
+			throw new Error(
+				`Failed to fetch WordPress static assets: ${response.status} ${response.statusText}`
+			);
+		}
+
+		await unzipFile(
+			php,
+			new File([await response.blob()], 'wordpress-static.zip'),
+			php.requestHandler.documentRoot,
+			false
+		);
+		// Clear the remote asset list to indicate that the assets are downloaded.
+		await php.writeFile(remoteAssetListPath, '');
+	} catch (e) {
+		logger.warn('Failed to download WordPress assets', e);
+	}
 }
 
 const apiEndpoint = new PlaygroundWorkerEndpoint(

packages/playground/wordpress-builds/build/Dockerfile

@@ -32,6 +32,7 @@ RUN rm -rf wordpress/wp-content/plugins/akismet
 RUN cp -r wordpress wordpress-static && \
     cd wordpress-static && \
     find ./ -name '*.php' -delete && \
+    find ./ -name '*.sqlite' -delete && \
     # Keep only the static files inside the directories like wp-admin or wp-content:
     find . -maxdepth 1 -type f -delete && \
     # Remove all empty directories
@@ -44,11 +45,19 @@ RUN rm -rf wordpress-static/wp-content/mu-plugins
 # to request a remote asset or delegate the request for a missing file to PHP.
 RUN find wordpress-static -type f | sed 's#^wordpress-static/##'> wordpress-remote-asset-paths
 
-# Make the remote asset listing available remotely so it can be downloaded 
+# Make the remote asset listing available remotely so it can be downloaded
 # directly in cases where an older minified WordPress build without this file
 # has been saved to browser storage.
 RUN cp wordpress-remote-asset-paths wordpress-static/
 
+# ZIP the static files
+RUN cd wordpress-static/ && \
+    zip -r ../wordpress-static.zip . && \
+    cd ..
+
+# Move ZIP to the public output directory
+RUN cp wordpress-static.zip wordpress-static/
+
 # Move the static files to the final output directory
 RUN mkdir /root/output/$OUT_FILENAME
 RUN mv wordpress-static/* /root/output/$OUT_FILENAME/
@@ -135,6 +144,6 @@ RUN cd wordpress && \
 # Build the final wp.zip file
 RUN mv wordpress /wordpress && \
     cp wordpress-remote-asset-paths /wordpress/ && \
+    cp wordpress-static.zip /wordpress/ && \
     cd /wordpress && \
     zip /root/output/$OUT_FILENAME.zip -r .
-

packages/playground/wordpress/src/boot.ts

@@ -219,7 +219,7 @@ async function isWordPressInstalled(php: PHP) {
 	return (
 		(
 			await php.run({
-				code: `<?php 
+				code: `<?php
 	require '${php.documentRoot}/wp-load.php';
 	echo is_blog_installed() ? '1' : '0';
 	`,

packages/playground/wordpress/src/index.ts

@@ -2,8 +2,8 @@ import { PHP, UniversalPHP } from '@php-wasm/universal';
 import { joinPaths, phpVar } from '@php-wasm/util';
 import { unzipFile } from '@wp-playground/common';
 export { bootWordPress } from './boot';
-export { getLoadedWordPressVersion } from './version-detect';
 
+export * from './version-detect';
 export * from './rewrite-rules';
 
 /**
@@ -18,7 +18,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
 	await php.writeFile(
 		'/internal/shared/preload/env.php',
 		`<?php
-    
+
         // Allow adding filters/actions prior to loading WordPress.
         // $function_to_add MUST be a string.
         function playground_add_filter( $tag, $function_to_add, $priority = 10, $accepted_args = 1 ) {
@@ -28,7 +28,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
         function playground_add_action( $tag, $function_to_add, $priority = 10, $accepted_args = 1 ) {
             playground_add_filter( $tag, $function_to_add, $priority, $accepted_args );
         }
-        
+
         // Load our mu-plugins after customer mu-plugins
         // NOTE: this means our mu-plugins can't use the muplugins_loaded action!
         playground_add_action( 'muplugins_loaded', 'playground_load_mu_plugins', 0 );
@@ -57,7 +57,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
             }
             return $redirect_url;
         } );
-		
+
         // Needed because gethostbyname( 'wordpress.org' ) returns
         // a private network IP address for some reason.
         add_filter( 'allowed_redirect_hosts', function( $deprecated = '' ) {
@@ -75,7 +75,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
         if(!file_exists(WP_CONTENT_DIR . '/fonts')) {
             mkdir(WP_CONTENT_DIR . '/fonts');
         }
-		
+
         $log_file = WP_CONTENT_DIR . '/debug.log';
         define('ERROR_LOG_FILE', $log_file);
         ini_set('error_log', $log_file);
@@ -88,7 +88,7 @@ export async function setupPlatformLevelMuPlugins(php: UniversalPHP) {
 	await php.writeFile(
 		'/internal/shared/preload/error-handler.php',
 		`<?php
-		(function() { 
+		(function() {
 			$playground_consts = [];
 			if(file_exists('/internal/shared/consts.json')) {
 				$playground_consts = @json_decode(file_get_contents('/internal/shared/consts.json'), true) ?: [];
@@ -204,7 +204,7 @@ export async function preloadSqliteIntegration(
 
 /**
  * Loads the SQLite integration plugin before WordPress is loaded
- * and without creating a drop-in "db.php" file. 
+ * and without creating a drop-in "db.php" file.
  *
  * Technically, it creates a global $wpdb object whose only two
  * purposes are to:

packages/playground/wordpress/src/version-detect.ts

@@ -15,7 +15,6 @@ export async function getLoadedWordPressVersion(
 	if (!versionString) {
 		throw new Error('Unable to read loaded WordPress version.');
 	}
-
 	return versionStringToLoadedWordPressVersion(versionString);
 }