Build/Test Tools: Expand performance test scenarios.

Adds new tests for localized sites as well as the dashboard.
Also amends Server-Timing output to measure memory usage in all scenarios.

Props swissspidy, joemcgill, flixos90, mukesh27, mamaduka.
See #59656.
Fixes #59815.

git-svn-id: https://develop.svn.wordpress.org/trunk@57083 602fd350-edb4-49c9-b593-d223f7449a82

.github/workflows/performance.yml

@@ -162,6 +162,12 @@ jobs:
         run: |
           npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
 
+      - name: Install additional languages
+        run: |
+          npm run env:cli -- language core install de_DE --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language plugin install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Install MU plugin
         run: |
           mkdir ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins

tests/performance/compare-results.js

@@ -23,7 +23,14 @@ const parseFile = ( fileName ) =>
 	);
 
 // The list of test suites to log.
-const testSuites = [ 'home-block-theme', 'home-classic-theme' ];
+const testSuites = [
+	'admin',
+	'admin-l10n',
+	'home-block-theme',
+	'home-block-theme-l10n',
+	'home-classic-theme',
+	'home-classic-theme-l10n',
+];
 
 // The current commit's results.
 const testResults = Object.fromEntries(
@@ -128,6 +135,23 @@ console.log( 'Performance Test Results\n' );
 
 console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' );
 
+/**
+ * Nicely formats a given value.
+ *
+ * @param {string} metric Metric.
+ * @param {number} value
+ */
+function formatValue( metric, value) {
+	if ( null === value ) {
+		return 'N/A';
+	}
+	if ( 'wpMemoryUsage' === metric ) {
+		return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`;
+	}
+
+	return `${ value.toFixed( 2 ) } ms`;
+}
+
 for ( const key of testSuites ) {
 	const current = testResults[ key ] || {};
 	const prev = prevResults[ key ] || {};
@@ -141,15 +165,15 @@ for ( const key of testSuites ) {
 
 	for ( const [ metric, values ] of Object.entries( current ) ) {
 		const value = median( values );
-		const prevValue = median( prev[ metric ] );
+		const prevValue = prev[ metric ] ? median( prev[ metric ] ) : null;
 
-		const delta = value - prevValue;
+		const delta = null !== prevValue ? value - prevValue : 0
 		const percentage = ( delta / value ) * 100;
 		rows.push( {
 			Metric: metric,
-			Before: `${ prevValue.toFixed( 2 ) } ms`,
+			Before: formatValue( metric, prevValue ),
-			After: `${ value.toFixed( 2 ) } ms`,
+			After: formatValue( metric, value ),
-			'Diff abs.': `${ delta.toFixed( 2 ) } ms`,
+			'Diff abs.': formatValue( metric, delta ),
 			'Diff %': `${ percentage.toFixed( 2 ) } %`,
 		} );
 	}

tests/performance/log-results.js

@@ -11,8 +11,12 @@ const { median } = require( './utils' );
 
 // The list of test suites to log.
 const testSuites = [
+	'admin',
+	'admin-l10n',
 	'home-block-theme',
+	'home-block-theme-l10n',
 	'home-classic-theme',
+	'home-classic-theme-l10n',
 ];
 
 // A list of results to parse based on test suites.

tests/performance/playwright.config.js

@@ -19,9 +19,7 @@ process.env.TEST_RUNS ??= '20';
 const config = defineConfig( {
 	...baseConfig,
 	globalSetup: require.resolve( './config/global-setup.js' ),
-	reporter: process.env.CI
+	reporter: [ [ 'list' ], [ './config/performance-reporter.js' ] ],
-		? './config/performance-reporter.js'
-		: [ [ 'list' ], [ './config/performance-reporter.js' ] ],
 	forbidOnly: !! process.env.CI,
 	workers: 1,
 	retries: 0,

tests/performance/results.js

@@ -8,8 +8,12 @@ const { join } = require( 'node:path' );
 const { median, getResultsFilename } = require( './utils' );
 
 const testSuites = [
+    'admin',
+    'admin-l10n',
     'home-classic-theme',
+    'home-classic-theme-l10n',
     'home-block-theme',
+    'home-block-theme-l10n',
 ];
 
 console.log( '\n>> 🎉 Results 🎉 \n' );

tests/performance/specs/admin-l10n.test.js

@@ -0,0 +1,52 @@
+/**
+ * WordPress dependencies
+ */
+import { test } from '@wordpress/e2e-test-utils-playwright';
+
+/**
+ * Internal dependencies
+ */
+import { camelCaseDashes } from '../utils';
+
+const results = {
+	timeToFirstByte: [],
+};
+
+test.describe( 'Admin (L10N)', () => {
+	test.beforeAll( async ( { requestUtils } ) => {
+		await requestUtils.activateTheme( 'twentytwentyone' );
+		await requestUtils.updateSiteSettings( {
+			language: 'de_DE',
+		} );
+	} );
+
+	test.afterAll( async ( { requestUtils }, testInfo ) => {
+		await testInfo.attach( 'results', {
+			body: JSON.stringify( results, null, 2 ),
+			contentType: 'application/json',
+		} );
+		await requestUtils.updateSiteSettings( {
+			language: '',
+		} );
+	} );
+
+	const iterations = Number( process.env.TEST_RUNS );
+	for ( let i = 1; i <= iterations; i++ ) {
+		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
+			admin,
+			metrics,
+		} ) => {
+			await admin.visitAdminPage( '/' );
+
+			const serverTiming = await metrics.getServerTiming();
+
+			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
+				results[ camelCaseDashes( key ) ] ??= [];
+				results[ camelCaseDashes( key ) ].push( value );
+			}
+
+			const ttfb = await metrics.getTimeToFirstByte();
+			results.timeToFirstByte.push( ttfb );
+		} );
+	}
+} );

tests/performance/specs/admin.test.js

@@ -0,0 +1,46 @@
+/**
+ * WordPress dependencies
+ */
+import { test } from '@wordpress/e2e-test-utils-playwright';
+
+/**
+ * Internal dependencies
+ */
+import { camelCaseDashes } from '../utils';
+
+const results = {
+	timeToFirstByte: [],
+};
+
+test.describe( 'Admin', () => {
+	test.beforeAll( async ( { requestUtils } ) => {
+		await requestUtils.activateTheme( 'twentytwentyone' );
+	} );
+
+	test.afterAll( async ( {}, testInfo ) => {
+		await testInfo.attach( 'results', {
+			body: JSON.stringify( results, null, 2 ),
+			contentType: 'application/json',
+		} );
+	} );
+
+	const iterations = Number( process.env.TEST_RUNS );
+	for ( let i = 1; i <= iterations; i++ ) {
+		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
+			admin,
+			metrics,
+		} ) => {
+			await admin.visitAdminPage( '/' );
+
+			const serverTiming = await metrics.getServerTiming();
+
+			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
+				results[ camelCaseDashes( key ) ] ??= [];
+				results[ camelCaseDashes( key ) ].push( value );
+			}
+
+			const ttfb = await metrics.getTimeToFirstByte();
+			results.timeToFirstByte.push( ttfb );
+		} );
+	}
+} );

tests/performance/specs/home-block-theme-l10n.test.js

@@ -0,0 +1,63 @@
+/**
+ * WordPress dependencies
+ */
+import { test } from '@wordpress/e2e-test-utils-playwright';
+
+/**
+ * Internal dependencies
+ */
+import { camelCaseDashes } from '../utils';
+
+const results = {
+	timeToFirstByte: [],
+	largestContentfulPaint: [],
+	lcpMinusTtfb: [],
+};
+
+test.describe( 'Front End - Twenty Twenty Three (L10N)', () => {
+	test.use( {
+		storageState: {}, // User will be logged out.
+	} );
+
+	test.beforeAll( async ( { requestUtils } ) => {
+		await requestUtils.activateTheme( 'twentytwentythree' );
+		await requestUtils.updateSiteSettings( {
+			language: 'de_DE',
+		} );
+	} );
+
+	test.afterAll( async ( { requestUtils }, testInfo ) => {
+		await testInfo.attach( 'results', {
+			body: JSON.stringify( results, null, 2 ),
+			contentType: 'application/json',
+		} );
+		await requestUtils.activateTheme( 'twentytwentyone' );
+		await requestUtils.updateSiteSettings( {
+			language: '',
+		} );
+	} );
+
+	const iterations = Number( process.env.TEST_RUNS );
+	for ( let i = 1; i <= iterations; i++ ) {
+		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
+			page,
+			metrics,
+		} ) => {
+			await page.goto( '/' );
+
+			const serverTiming = await metrics.getServerTiming();
+
+			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
+				results[ camelCaseDashes( key ) ] ??= [];
+				results[ camelCaseDashes( key ) ].push( value );
+			}
+
+			const ttfb = await metrics.getTimeToFirstByte();
+			const lcp = await metrics.getLargestContentfulPaint();
+
+			results.largestContentfulPaint.push( lcp );
+			results.timeToFirstByte.push( ttfb );
+			results.lcpMinusTtfb.push( lcp - ttfb );
+		} );
+	}
+} );

tests/performance/specs/home-block-theme.test.js

@@ -41,7 +41,7 @@ test.describe( 'Front End - Twenty Twenty Three', () => {
 
 			const serverTiming = await metrics.getServerTiming();
 
-			for ( const [key, value] of Object.entries( serverTiming ) ) {
+			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
 				results[ camelCaseDashes( key ) ] ??= [];
 				results[ camelCaseDashes( key ) ].push( value );
 			}

tests/performance/specs/home-classic-theme-l10n.test.js

@@ -0,0 +1,62 @@
+/**
+ * WordPress dependencies
+ */
+import { test } from '@wordpress/e2e-test-utils-playwright';
+
+/**
+ * Internal dependencies
+ */
+import { camelCaseDashes } from '../utils';
+
+const results = {
+	timeToFirstByte: [],
+	largestContentfulPaint: [],
+	lcpMinusTtfb: [],
+};
+
+test.describe( 'Front End - Twenty Twenty One (L10N)', () => {
+	test.use( {
+		storageState: {}, // User will be logged out.
+	} );
+
+	test.beforeAll( async ( { requestUtils } ) => {
+		await requestUtils.activateTheme( 'twentytwentyone' );
+		await requestUtils.updateSiteSettings( {
+			language: 'de_DE',
+		} );
+	} );
+
+	test.afterAll( async ( { requestUtils }, testInfo ) => {
+		await testInfo.attach( 'results', {
+			body: JSON.stringify( results, null, 2 ),
+			contentType: 'application/json',
+		} );
+		await requestUtils.updateSiteSettings( {
+			language: '',
+		} );
+	} );
+
+	const iterations = Number( process.env.TEST_RUNS );
+	for ( let i = 1; i <= iterations; i++ ) {
+		test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
+			page,
+			metrics,
+		} ) => {
+			await page.goto( '/' );
+
+			const serverTiming = await metrics.getServerTiming();
+
+			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
+				results[ camelCaseDashes( key ) ] ??= [];
+				results[ camelCaseDashes( key ) ].push( value );
+			}
+
+			const ttfb = await metrics.getTimeToFirstByte();
+			const lcp = await metrics.getLargestContentfulPaint();
+
+			results.largestContentfulPaint.push( lcp );
+			results.timeToFirstByte.push( ttfb );
+			results.lcpMinusTtfb.push( lcp - ttfb );
+		} );
+	}
+} );

tests/performance/specs/home-classic-theme.test.js

@@ -40,7 +40,7 @@ test.describe( 'Front End - Twenty Twenty One', () => {
 
 			const serverTiming = await metrics.getServerTiming();
 
-			for (const [key, value] of Object.entries( serverTiming ) ) {
+			for ( const [ key, value ] of Object.entries( serverTiming ) ) {
 				results[ camelCaseDashes( key ) ] ??= [];
 				results[ camelCaseDashes( key ) ].push( value );
 			}

tests/performance/wp-content/mu-plugins/server-timing.php

@@ -25,6 +25,13 @@ add_filter(
 
 				$server_timing_values['total'] = $server_timing_values['before-template'] + $server_timing_values['template'];
 
+				/*
+				 * While values passed via Server-Timing are intended to be durations,
+				 * any numeric value can actually be passed.
+				 * This is a nice little trick as it allows to easily get this information in JS.
+				 */
+				$server_timing_values['memory-usage'] = memory_get_usage();
+
 				$header_values = array();
 				foreach ( $server_timing_values as $slug => $value ) {
 					if ( is_float( $value ) ) {