Build/Test Tools: Expand performance tests setup.

Run tests against Multisite (possible since [58097]) and on single post pages. Also improve cache flushes/resets between iterations.

Props swissspidy, flixos90, desrosj, mukesh27.
Fixes #62725.

git-svn-id: https://develop.svn.wordpress.org/trunk@59577 602fd350-edb4-49c9-b593-d223f7449a82

.github/workflows/performance.yml

@@ -32,7 +32,7 @@ permissions: {}
 jobs:
   # Runs the performance test suite.
   performance:
-    name: Performance tests ${{ matrix.memcached && '(with memcached)' || '' }}
+    name: ${{ matrix.multisite && 'Multisite' || 'Single site' }}
     uses: WordPress/wordpress-develop/.github/workflows/reusable-performance.yml@trunk
     permissions:
       contents: read
@@ -41,8 +41,10 @@ jobs:
       fail-fast: false
       matrix:
         memcached: [ true, false ]
+        multisite: [ true, false ]
     with:
       memcached: ${{ matrix.memcached }}
+      multisite: ${{ matrix.multisite }}
     secrets:
       CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }}
 

.github/workflows/reusable-performance.yml

@@ -1,7 +1,7 @@
 ##
 # A reusable workflow that runs the performance test suite.
 ##
-name: Performance Tests
+name: Run performance Tests
 
 on:
   workflow_call:
@@ -26,6 +26,11 @@ on:
         required: false
         type: 'boolean'
         default: false
+      multisite:
+        description: 'Whether to use Multisite.'
+        required: false
+        type: 'boolean'
+        default: false
     secrets:
       CODEVITALS_PROJECT_TOKEN:
         description: 'The authorization token for https://www.codevitals.run/project/wordpress.'
@@ -53,6 +58,7 @@ env:
 
   LOCAL_PHP_MEMCACHED: ${{ inputs.memcached }}
   LOCAL_PHP: ${{ inputs.php-version }}${{ 'latest' != inputs.php-version && '-fpm' || '' }}
+  LOCAL_MULTISITE: ${{ inputs.multisite }}
 
 jobs:
   # Performs the following steps:
@@ -65,9 +71,11 @@ jobs:
   # - Install Playwright browsers.
   # - Build WordPress.
   # - Start Docker environment.
+  # - Install object cache drop-in.
   # - Log running Docker containers.
   # - Docker debug information.
   # - Install WordPress.
+  # - Enable themes on Multisite.
   # - Install WordPress Importer plugin.
   # - Import mock data.
   # - Deactivate WordPress Importer plugin.
@@ -98,7 +106,7 @@ jobs:
   # - Publish performance results.
   # - Ensure version-controlled files are not modified or deleted.
   performance:
-    name: Run tests
+    name: ${{ inputs.multisite && 'Multisite' || 'Single site' }} / ${{ inputs.memcached && 'Memcached' || 'Default' }}
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -166,6 +174,14 @@ jobs:
       - name: Install WordPress
         run: npm run env:install
 
+      - name: Enable themes on Multisite
+        if: ${{ inputs.multisite }}
+        run: |
+          npm run env:cli -- theme enable twentytwentyone --network --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- theme enable twentytwentythree --network --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- theme enable twentytwentyfour --network --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- theme enable twentytwentyfive --network --path=/var/www/${{ env.LOCAL_DIR }}
+
       - name: Install WordPress Importer plugin
         run: npm run env:cli -- plugin install wordpress-importer --activate --path=/var/www/${{ env.LOCAL_DIR }}
 
@@ -290,7 +306,7 @@ jobs:
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         if: always()
         with:
-          name: performance-artifacts${{ inputs.memcached && '-memcached' || '' }}-${{ github.run_id }}
+          name: performance-artifacts${{ inputs.multisite && '-multisite' || '' }}${{ inputs.memcached && '-memcached' || '' }}-${{ github.run_id }}
           path: artifacts
           if-no-files-found: ignore
           include-hidden-files: true
@@ -303,7 +319,7 @@ jobs:
 
       - name: Set the base sha
         # Only needed when publishing results.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! inputs.memcached }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! inputs.memcached && ! inputs.multisite }}
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         id: base-sha
         with:
@@ -314,7 +330,7 @@ jobs:
 
       - name: Set commit details
         # Only needed when publishing results.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! inputs.memcached }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! inputs.memcached && ! inputs.multisite }}
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         id: commit-timestamp
         with:
@@ -325,7 +341,7 @@ jobs:
 
       - name: Publish performance results
         # Only publish results on pushes to trunk.
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! inputs.memcached }}
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && ! inputs.memcached && ! inputs.multisite }}
         env:
           BASE_SHA: ${{ steps.base-sha.outputs.result }}
           COMMITTED_AT: ${{ steps.commit-timestamp.outputs.result }}

tests/performance/compare-results.js

@@ -96,6 +96,8 @@ if ( process.env.GITHUB_SHA ) {
 	);
 }
 
+summaryMarkdown += `<details><summary>Results</summary>`;
+
 for ( const { title, results } of afterStats ) {
 	const prevStat = beforeStats.find( ( s ) => s.title === title );
 
@@ -143,10 +145,12 @@ for ( const { title, results } of afterStats ) {
 		console.log( '(no results)' );
 	}
 
-	summaryMarkdown += `**${ title }**\n\n`;
+	summaryMarkdown += `<b>${ title }</b>\n\n`;
 	summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
 }
 
+summaryMarkdown += `</details>`;
+
 writeFileSync(
 	join( process.env.WP_ARTIFACTS_PATH, '/performance-results.md' ),
 	summaryMarkdown

tests/performance/log-results.js

@@ -18,12 +18,18 @@ const { median, parseFile, accumulateValues } = require( './utils' );
 const testSuiteMap = {
 	'Admin › Locale: en_US': 'admin',
 	'Admin › Locale: de_DE': 'admin-l10n',
-	'Front End › Theme: twentytwentyone, Locale: en_US': 'home-classic-theme',
-	'Front End › Theme: twentytwentyone, Locale: de_DE':
+	'Homepage › Theme: twentytwentyone, Locale: en_US': 'home-classic-theme',
+	'Homepage › Theme: twentytwentyone, Locale: de_DE':
 		'home-classic-theme-l10n',
-	'Front End › Theme: twentytwentythree, Locale: en_US': 'home-block-theme',
-	'Front End › Theme: twentytwentythree, Locale: de_DE':
+	'Homepage › Theme: twentytwentythree, Locale: en_US': 'home-block-theme',
+	'Homepage › Theme: twentytwentythree, Locale: de_DE':
 		'home-block-theme-l10n',
+	'Homepage › Theme: twentytwentyfour, Locale: en_US': 'home-twentytwentyfour',
+	'Homepage › Theme: twentytwentyfour, Locale: de_DE':
+		'home-twentytwentyfour-l10n',
+	'Homepage › Theme: twentytwentyfive, Locale: en_US': 'home-twentytwentyfive',
+	'Homepage › Theme: twentytwentyfive, Locale: de_DE':
+		'home-twentytwentyfive-l10n',
 };
 
 /**

tests/performance/specs/admin.test.js

@@ -6,14 +6,12 @@ import { test } from '@wordpress/e2e-test-utils-playwright';
 /**
  * Internal dependencies
  */
-import { camelCaseDashes } from '../utils';
+import { camelCaseDashes, locales } from '../utils';
 
 const results = {
 	timeToFirstByte: [],
 };
 
-const locales = [ 'en_US', 'de_DE' ];
-
 test.describe( 'Admin', () => {
 	for ( const locale of locales ) {
 		test.describe( `Locale: ${ locale }`, () => {
@@ -47,9 +45,14 @@ test.describe( 'Admin', () => {
 			const iterations = Number( process.env.TEST_RUNS );
 			for ( let i = 1; i <= iterations; i++ ) {
 				test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
+					page,
 					admin,
 					metrics,
 				} ) => {
+					// Clear caches using the clear-cache.php mu-plugin. Not actually loading the page.
+					await page.goto( '/?clear_cache' );
+
+					// This is the actual page to test.
 					await admin.visitAdminPage( '/' );
 
 					const serverTiming = await metrics.getServerTiming();

tests/performance/specs/home.test.js

@@ -6,7 +6,7 @@ import { test } from '@wordpress/e2e-test-utils-playwright';
 /**
  * Internal dependencies
  */
-import { camelCaseDashes } from '../utils';
+import { camelCaseDashes, themes, locales } from '../utils';
 
 const results = {
 	timeToFirstByte: [],
@@ -14,11 +14,7 @@ const results = {
 	lcpMinusTtfb: [],
 };
 
-const themes = [ 'twentytwentyone', 'twentytwentythree', 'twentytwentyfour', 'twentytwentyfive' ];
-
-const locales = [ 'en_US', 'de_DE' ];
-
-test.describe( 'Front End', () => {
+test.describe( 'Homepage', () => {
 	test.use( {
 		storageState: {}, // User will be logged out.
 	} );
@@ -54,6 +50,10 @@ test.describe( 'Front End', () => {
 						page,
 						metrics,
 					} ) => {
+						// Clear caches using the clear-cache.php mu-plugin. Not actually loading the page.
+						await page.goto( '/?clear_cache' );
+
+						// This is the actual page to test.
 						await page.goto( '/' );
 
 						const serverTiming = await metrics.getServerTiming();

tests/performance/specs/single-post.test.js

@@ -0,0 +1,79 @@
+/**
+ * WordPress dependencies
+ */
+import { test } from '@wordpress/e2e-test-utils-playwright';
+
+/**
+ * Internal dependencies
+ */
+import { camelCaseDashes, themes, locales } from '../utils';
+
+const results = {
+	timeToFirstByte: [],
+	largestContentfulPaint: [],
+	lcpMinusTtfb: [],
+};
+
+test.describe( 'Single Post', () => {
+	test.use( {
+		storageState: {}, // User will be logged out.
+	} );
+
+	for ( const theme of themes ) {
+		for ( const locale of locales ) {
+			test.describe( `Theme: ${ theme }, Locale: ${ locale }`, () => {
+				test.beforeAll( async ( { requestUtils } ) => {
+					await requestUtils.activateTheme( theme );
+					await requestUtils.updateSiteSettings( {
+						language: 'en_US' === locale ? '' : locale,
+					} );
+				} );
+
+				test.afterAll( async ( { requestUtils }, testInfo ) => {
+					await testInfo.attach( 'results', {
+						body: JSON.stringify( results, null, 2 ),
+						contentType: 'application/json',
+					} );
+
+					await requestUtils.updateSiteSettings( {
+						language: '',
+					} );
+
+					results.largestContentfulPaint = [];
+					results.timeToFirstByte = [];
+					results.lcpMinusTtfb = [];
+				} );
+
+				const iterations = Number( process.env.TEST_RUNS );
+				for ( let i = 1; i <= iterations; i++ ) {
+					test( `Measure load time metrics (${ i } of ${ iterations })`, async ( {
+						page,
+						metrics,
+					} ) => {
+						// Clear caches using the clear-cache.php mu-plugin. Not actually loading the page.
+						await page.goto( '/?clear_cache' );
+
+						// This is the actual page to test.
+						await page.goto( '/2018/11/03/block-image/' );
+
+						const serverTiming = await metrics.getServerTiming();
+
+						for ( const [ key, value ] of Object.entries(
+							serverTiming
+						) ) {
+							results[ camelCaseDashes( key ) ] ??= [];
+							results[ camelCaseDashes( key ) ].push( value );
+						}
+
+						const ttfb = await metrics.getTimeToFirstByte();
+						const lcp = await metrics.getLargestContentfulPaint();
+
+						results.largestContentfulPaint.push( lcp );
+						results.timeToFirstByte.push( ttfb );
+						results.lcpMinusTtfb.push( lcp - ttfb );
+					} );
+				}
+			} );
+		}
+	}
+} );

tests/performance/utils.js

@@ -6,6 +6,10 @@ const { join } = require( 'node:path' );
 
 process.env.WP_ARTIFACTS_PATH ??= join( process.cwd(), 'artifacts' );
 
+const locales = [ 'en_US', 'de_DE' ];
+
+const themes = [ 'twentytwentyone', 'twentytwentythree', 'twentytwentyfour', 'twentytwentyfive' ];
+
 /**
  * Parse test files into JSON objects.
  *
@@ -189,4 +193,6 @@ module.exports = {
 	standardDeviation,
 	medianAbsoluteDeviation,
 	accumulateValues,
+	themes,
+	locales,
 };

tests/performance/wp-content/mu-plugins/clear-cache.php

@@ -0,0 +1,27 @@
+<?php
+
+add_action(
+	'plugins_loaded',
+	static function () {
+		if ( isset( $_GET['clear_cache'] ) ) {
+			if ( function_exists( 'opcache_reset' ) ) {
+				opcache_reset();
+			}
+
+			if ( function_exists( 'apcu_clear_cache' ) ) {
+				apcu_clear_cache();
+			}
+
+			wp_cache_flush();
+
+			delete_expired_transients( true );
+
+			clearstatcache( true );
+
+			status_header( 202 );
+
+			die;
+		}
+	},
+	1
+);