chore: speed up CI a bit (hopefully)

.github/workflows/ci.yml

@@ -9,6 +9,7 @@ on:
 env:
   # we call `pnpm playwright install` instead
   PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
+  PW_DISABLE_TS_ESM: '1'
 
 # cancel in-progress runs on new commits to same PR (gitub.event.number)
 concurrency:
@@ -60,6 +61,10 @@ jobs:
       - run: pnpm install --frozen-lockfile
       - run: pnpm playwright install ${{ matrix.e2e-browser }}
       - run: pnpm test
+
+      - name: Print flaky test report
+        run: node scripts/print-flaky-test-report.js
+
       - name: Archive test results
         if: failure()
         shell: bash
@@ -115,6 +120,10 @@ jobs:
       - run: pnpm install --frozen-lockfile
       - run: pnpm playwright install ${{ matrix.e2e-browser }}
       - run: pnpm test:cross-platform:${{ matrix.mode }}
+
+      - name: Print flaky test report
+        run: node scripts/print-flaky-test-report.js
+
       - name: Archive test results
         if: failure()
         shell: bash

packages/kit/test/apps/basics/src/global.d.ts

@@ -6,6 +6,7 @@ declare global {
 		mounted: number;
 		fulfil_navigation: (value: any) => void;
 		promise: Promise<any>;
+		update_state: number;
 		PUBLIC_DYNAMIC: string;
 	}
 }

packages/kit/test/apps/basics/src/routes/load/cache-control/bust/+page.svelte

@@ -11,5 +11,5 @@
 	}
 </script>
 
-<p>Count is {data.count}</p>
+<p class="counter">Count is {data.count}</p>
 <button on:click={update}>update</button>

packages/kit/test/apps/basics/src/routes/load/cache-control/bust/count/+server.js

@@ -1,8 +1,10 @@
 import { json } from '@sveltejs/kit';
-import { count } from '../state.js';
 
-export function GET({ setHeaders }) {
+export function GET({ setHeaders, cookies }) {
 	setHeaders({ 'cache-control': 'public, max-age=4', age: '2' });
+
+	const count = +(cookies.get('cache-control-bust-count') ?? 0);
+
 	return json({ count });
 }
 

packages/kit/test/apps/basics/src/routes/load/cache-control/bust/increment/+server.js

@@ -1,7 +1,7 @@
 import { json } from '@sveltejs/kit';
-import { increment } from '../state.js';
 
-export function GET() {
-	increment();
+export function GET({ cookies }) {
+	cookies.set('cache-control-bust-count', +(cookies.get('cache-control-bust-count') ?? 0) + 1 + '');
+
 	return json({});
 }

packages/kit/test/apps/basics/src/routes/load/cache-control/default/+page.svelte

@@ -5,10 +5,12 @@
 	export let data;
 
 	async function update() {
+		window.update_state += 1;
 		await fetch('/load/cache-control/default/increment');
-		invalidate('/load/cache-control/default/count');
+		await invalidate('/load/cache-control/default/count');
+		window.update_state += 1;
 	}
 </script>
 
-<p>Count is {data.count}</p>
+<p class="counter">Count is {data.count}</p>
 <button on:click={update}>update</button>

packages/kit/test/apps/basics/src/routes/load/cache-control/default/count/+server.js

@@ -1,8 +1,10 @@
 import { json } from '@sveltejs/kit';
-import { count } from '../state.js';
 
-export function GET({ setHeaders }) {
+export function GET({ setHeaders, cookies }) {
 	setHeaders({ 'cache-control': 'public, max-age=4', age: '2' });
+
+	const count = +(cookies.get('cache-control-default-count') ?? 0);
+
 	return json({ count });
 }
 

packages/kit/test/apps/basics/src/routes/load/cache-control/default/increment/+server.js

@@ -1,7 +1,10 @@
 import { json } from '@sveltejs/kit';
-import { increment } from '../state.js';
 
-export function GET() {
-	increment();
+export function GET({ cookies }) {
+	cookies.set(
+		'cache-control-default-count',
+		+(cookies.get('cache-control-default-count') ?? 0) + 1 + ''
+	);
+
 	return json({});
 }

packages/kit/test/apps/basics/src/routes/load/cache-control/force/+page.svelte

@@ -10,5 +10,5 @@
 	}
 </script>
 
-<p>Count is {data.count}</p>
+<p class="counter">Count is {data.count}</p>
 <button on:click={update}>update</button>

packages/kit/test/apps/basics/src/routes/load/cache-control/force/count/+server.js

@@ -1,8 +1,10 @@
 import { json } from '@sveltejs/kit';
-import { count } from '../state.js';
 
-export function GET({ setHeaders }) {
+export function GET({ setHeaders, cookies }) {
 	setHeaders({ 'cache-control': 'public, max-age=4', age: '2' });
+
+	const count = +(cookies.get('cache-control-force-count') ?? 0);
+
 	return json({ count });
 }
 

packages/kit/test/apps/basics/src/routes/load/cache-control/force/increment/+server.js

@@ -1,7 +1,10 @@
 import { json } from '@sveltejs/kit';
-import { increment } from '../state.js';
 
-export function GET() {
-	increment();
+export function GET({ cookies }) {
+	cookies.set(
+		'cache-control-force-count',
+		+(cookies.get('cache-control-force-count') ?? 0) + 1 + ''
+	);
+
 	return json({});
 }

packages/kit/test/apps/basics/test/client.test.js

@@ -143,35 +143,43 @@ test.describe('Load', () => {
 	});
 
 	test('load does not call fetch if max-age allows it', async ({ page }) => {
-		page.addInitScript(`
+		// update_state is used to ensure we start waiting at the right time
+		await page.addInitScript(`
 			window.now = 0;
+			window.update_state = 0;
 			window.performance.now = () => now;
 		`);
 
 		await page.goto('/load/cache-control/default');
-		await expect(page.getByText('Count is 0')).toBeVisible();
-		await page.locator('button').click();
-		await page.waitForLoadState('networkidle');
-		await expect(page.getByText('Count is 0')).toBeVisible();
 
-		await page.evaluate(() => (window.now = 2500));
+		const button = page.locator('button');
+		const p = page.locator('p.counter');
 
-		await page.locator('button').click();
-		await expect(page.getByText('Count is 2')).toBeVisible();
+		await button.click();
+		await page.waitForFunction('window.update_state === 2');
+		await expect(p).toHaveText('Count is 0');
+
+		await page.evaluate('window.now += 2000');
+
+		await button.click();
+		await page.waitForFunction('window.update_state === 4');
+		await expect(p).toHaveText('Count is 2');
 	});
 
 	test('load does ignore ttl if fetch cache options says so', async ({ page }) => {
 		await page.goto('/load/cache-control/force');
-		await expect(page.getByText('Count is 0')).toBeVisible();
+		const p = page.locator('p.counter');
+		await expect(p).toHaveText('Count is 0');
 		await page.locator('button').click();
-		await expect(page.getByText('Count is 1')).toBeVisible();
+		await expect(p).toHaveText('Count is 1');
 	});
 
 	test('load busts cache if non-GET request to resource is made', async ({ page }) => {
 		await page.goto('/load/cache-control/bust');
-		await expect(page.getByText('Count is 0')).toBeVisible();
+		const p = page.locator('p.counter');
+		await expect(p).toHaveText('Count is 0');
 		await page.locator('button').click();
-		await expect(page.getByText('Count is 1')).toBeVisible();
+		await expect(p).toHaveText('Count is 1');
 	});
 
 	test('__data.json has cache-control: private, no-store', async ({ page, clicknav }) => {

packages/kit/test/apps/basics/test/cross-platform/client.test.js

@@ -75,7 +75,10 @@ test.describe('a11y', () => {
 			})
 		).toBe(1);
 
-		await clicknav('[href="/selection/b"]');
+		await clicknav('[href="/selection/b"]', {
+			waitUntil: 'networkidle'
+		});
+
 		expect(
 			await page.evaluate(() => {
 				const selection = getSelection();
@@ -98,16 +101,21 @@ test.describe('a11y', () => {
 	});
 
 	test('autofocus from previous page is ignored', async ({ page, clicknav }) => {
-		page.addInitScript(`
+		await page.addInitScript(`
 			window.active = null;
 			window.addEventListener('focusin', () => window.active = document.activeElement);
 		`);
 
 		await page.goto('/accessibility/autofocus/a');
 		await clicknav('[href="/"]');
 
-		expect(await page.evaluate(() => (window.active || {}).nodeName)).toBe('BODY');
-		expect(await page.evaluate(() => (document.activeElement || {}).nodeName)).toBe('BODY');
+		expect(
+			await page.evaluate(
+				// @ts-expect-error
+				() => window.active?.nodeName
+			)
+		).toBe('BODY');
+		expect(await page.evaluate(() => document.activeElement?.nodeName)).toBe('BODY');
 	});
 });
 
@@ -302,13 +310,12 @@ test.describe('Scrolling', () => {
 		await page.locator('#scroll-anchor').click();
 		const originalScrollY = /** @type {number} */ (await page.evaluate(() => scrollY));
 		await clicknav('#routing-page');
-		await page.goBack();
-		await page.waitForLoadState('networkidle');
+		await page.goBack({ waitUntil: 'networkidle' });
+
 		expect(page.url()).toBe(baseURL + '/anchor#last-anchor-2');
 		expect(await page.evaluate(() => scrollY)).toEqual(originalScrollY);
 
-		await page.goBack();
-		await page.waitForLoadState('networkidle');
+		await page.goBack({ waitUntil: 'networkidle' });
 
 		expect(page.url()).toBe(baseURL + '/anchor');
 		expect(await page.evaluate(() => scrollY)).toEqual(0);
@@ -507,7 +514,9 @@ test.describe.serial('Errors', () => {
 
 test.describe('Prefetching', () => {
 	test('prefetches programmatically', async ({ baseURL, page, app }) => {
-		await page.goto('/routing/a');
+		await page.goto('/routing/a', {
+			waitUntil: 'networkidle'
+		});
 
 		/** @type {string[]} */
 		let requests = [];
@@ -632,8 +641,9 @@ test.describe('Routing', () => {
 		await page.locator('[href="#hash-target"]').click();
 		await clicknav('[href="/routing/hashes/b"]');
 
+		await expect(page.locator('h1')).toHaveText('b');
 		await page.goBack();
-		expect(await page.textContent('h1')).toBe('a');
+		await expect(page.locator('h1')).toHaveText('a');
 	});
 
 	test('replaces state if the data-sveltekit-replacestate router option is specified for the hash link', async ({
@@ -684,7 +694,9 @@ test.describe('Routing', () => {
 	});
 
 	test('responds to <form method="GET"> submission without reload', async ({ page }) => {
-		await page.goto('/routing/form-get');
+		await page.goto('/routing/form-get', {
+			waitUntil: 'networkidle'
+		});
 		expect(await page.textContent('h1')).toBe('...');
 		expect(await page.textContent('h2')).toBe('enter');
 		expect(await page.textContent('h3')).toBe('...');

packages/kit/test/apps/basics/test/cross-platform/test.js

@@ -738,21 +738,16 @@ test.describe('Routing', () => {
 		await page.goto('/routing');
 		await clicknav('[href="/routing/a"]');
 
-		await page.goBack();
-		await page.waitForLoadState('networkidle');
+		await page.goBack({ waitUntil: 'networkidle' });
 		expect(await page.textContent('h1')).toBe('Great success!');
 	});
 
 	test('focus works if page load has hash', async ({ page, browserName }) => {
-		await page.goto('/routing/hashes/target#p2');
+		await page.goto('/routing/hashes/target#p2', { waitUntil: 'networkidle' });
 
 		await page.keyboard.press(browserName === 'webkit' ? 'Alt+Tab' : 'Tab');
-		await page.waitForTimeout(50); // give browser a bit of time to complete the native behavior of the key press
-		expect(
-			await page.evaluate(
-				() => document.activeElement?.textContent || 'ERROR: document.activeElement not set'
-			)
-		).toBe('next focus element');
+
+		await page.waitForSelector('button:focus');
 	});
 
 	test('focus works when navigating to a hash on the same page', async ({ page, browserName }) => {

packages/kit/test/apps/basics/test/test.js

@@ -500,7 +500,7 @@ test.describe('Load', () => {
 	}) => {
 		if (javaScriptEnabled) {
 			await page.goto('/prerendering/prerendered-endpoint');
-			await page.click('a', { noWaitAfter: true });
+			await page.click('a');
 		} else {
 			await page.goto('/prerendering/prerendered-endpoint/from-handle-hook');
 		}

packages/kit/test/apps/embed/test/test.js

@@ -10,8 +10,8 @@ test.describe('embed', () => {
 		await page.goto('/embed');
 
 		if (javaScriptEnabled) {
-			expect(await page.textContent('[data-testid="a"]')).toBe('a (browser)');
-			expect(await page.textContent('[data-testid="b"]')).toBe('b (browser)');
+			await expect(page.getByTestId('a')).toHaveText('a (browser)');
+			await expect(page.getByTestId('b')).toHaveText('b (browser)');
 		} else {
 			expect(await page.textContent('[data-testid="a"]')).toBe('a (server)');
 			expect(await page.textContent('[data-testid="b"]')).toBe('b (server)');

packages/kit/test/github-flaky-warning-reporter.js

@@ -1,3 +1,5 @@
+import { appendFileSync } from 'node:fs';
+
 /**
  * @class
  * @implements {import('@playwright/test/reporter').Reporter}
@@ -24,12 +26,17 @@ export default class GithubFlakyWarningReporter {
 	}
 
 	onEnd() {
-		this._flaky.forEach(({ file, line, title, message }) => {
-			console.log(`::warning file=${file},line=${line},title=${title}::${message}`);
-		});
+		const output = this._flaky
+			.map(
+				({ file, line, title, message }) =>
+					`::warning file=${file},line=${line},title=${title}::${message}\n`
+			)
+			.join('');
+
+		appendFileSync(new URL('../../../_tmp_flaky_test_output.txt', import.meta.url), output);
 	}
 
 	printsToStdio() {
-		return true;
+		return false;
 	}
 }