Build/Test Tools: Add a performance measurement workflow.

This adds a new GitHub Action workflow that measures a set of performance metrics on every commit, so we can track changes in the performance of WordPress over time and more easily identify changes that are responsible for significant performance improvements or regressions during development cycles.

The workflow measures the homepage of a classic theme (Twenty Twenty-One) and a block theme (Twenty Twenty-Three) set up with demo content from the Theme Test Data project. Using the e2e testing framework, it makes 20 requests and records the median value of the following Server Timing metrics, generated by an mu-plugin installed as part of this workflow:

- Total server response time
- Server time before templates are loaded
- Server time during template rendering

In addition to measuring the performance metrics of the current commit, it also records performance metrics of a consistent version of WordPress (6.1.1) to be used as a baseline measurement in order to remove variance caused by the GitHub workers themselves from our reporting.

The measurements are collected and displayed at https://www.codevitals.run/project/wordpress.

Props adamsilverstein, mukesh27, flixos90, youknowriad, oandregal, desrosj, costdev, swissspidy.
Fixes #57687.


git-svn-id: https://develop.svn.wordpress.org/trunk@55459 602fd350-edb4-49c9-b593-d223f7449a82

.github/workflows/performance.yml

@@ -0,0 +1,231 @@
+name: Performance Tests
+
+on:
+  push:
+    branches:
+      - trunk
+      - '6.[2-9]'
+      - '[7-9].[0-9]'
+    tags:
+      - '[0-9]+.[0-9]'
+      - '[0-9]+.[0-9].[0-9]+'
+      - '![45].[0-9].[0-9]+'
+      - '!6.[01].[0-9]+'
+  pull_request:
+    branches:
+      - trunk
+      - '6.[2-9]'
+      - '[7-9].[0-9]'
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+concurrency:
+  # The concurrency group contains the workflow name and the branch name for pull requests
+  # or the commit hash for any other events.
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+env:
+  # This workflow takes two sets of measurements — one for the current commit,
+  # and another against a consistent version that is used as a baseline measurement.
+  # This is done to isolate variance in measurements caused by the GitHub runners
+  # from differences caused by code changes between commits. The BASE_TAG value here
+  # represents the version being used for baseline measurements. It should only be
+  # changed if we want to normalize results against a different baseline.
+  BASE_TAG: '6.1.1'
+  LOCAL_DIR: build
+
+jobs:
+  # Runs the performance test suite.
+  #
+  # Performs the following steps:
+  # - Configure environment variables.
+  # - Checkout repository.
+  # - Set up Node.js.
+  # - Log debug information.
+  # - Install npm dependencies.
+  # - Build WordPress.
+  # - Start Docker environment.
+  # - Log running Docker containers.
+  # - Docker debug information.
+  # - Install WordPress.
+  # - Install WordPress Importer plugin.
+  # - Import mock data.
+  # - Update permalink structure.
+  # - Install MU plugin.
+  # - Run performance tests (current commit).
+  # - Print performance tests results.
+  # - Set the environment to the baseline version.
+  # - Run baseline performance tests.
+  # - Print base line performance tests results.
+  # - Set the base sha.
+  # - Set commit details.
+  # - Publish performance results.
+  # - Ensure version-controlled files are not modified or deleted.
+  # - Dispatch workflow run.
+  performance:
+    name: Run performance tests
+    runs-on: ubuntu-latest
+    if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' }}
+
+    steps:
+      - name: Configure environment variables
+        run: |
+          echo "PHP_FPM_UID=$(id -u)" >> $GITHUB_ENV
+          echo "PHP_FPM_GID=$(id -g)" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
+        with:
+          node-version-file: '.nvmrc'
+          cache: npm
+
+      - name: Log debug information
+        run: |
+          npm --version
+          node --version
+          curl --version
+          git --version
+          svn --version
+          locale -a
+
+      - name: Install npm dependencies
+        run: npm ci
+
+      - name: Build WordPress
+        run: npm run build
+
+      - name: Start Docker environment
+        run: |
+          npm run env:start
+
+      - name: Log running Docker containers
+        run: docker ps -a
+
+      - name: Docker debug information
+        run: |
+          docker -v
+          docker-compose -v
+          docker-compose run --rm mysql mysql --version
+          docker-compose run --rm php php --version
+          docker-compose run --rm php php -m
+          docker-compose run --rm php php -i
+          docker-compose run --rm php locale -a
+
+      - name: Install WordPress
+        run: npm run env:install
+
+      - name: Install WordPress Importer plugin
+        run: npm run env:cli -- plugin install wordpress-importer --activate --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Import mock data
+        run: |
+          curl -O https://raw.githubusercontent.com/WPTT/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml
+          npm run env:cli -- import themeunittestdata.wordpress.xml --authors=create --path=/var/www/${{ env.LOCAL_DIR }}
+          rm themeunittestdata.wordpress.xml
+
+      - name: Update permalink structure
+        run: |
+          npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Install MU plugin
+        run: |
+          mkdir ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins
+          cp ./tests/performance/wp-content/mu-plugins/server-timing.php ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins/server-timing.php
+
+      - name: Run performance tests (current commit)
+        run: npm run test:performance
+
+      - name: Print performance tests results
+        run: "node ./tests/performance/results.js"
+
+      - name: Set the environment to the baseline version
+        run: |
+          npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }}
+          npm run env:cli -- core version --path=/var/www/${{ env.LOCAL_DIR }}
+
+      - name: Run baseline performance tests
+        run: npm run test:performance -- --prefix=base
+
+      - name: Print base line performance tests results
+        run: "node ./tests/performance/results.js --prefix=base"
+
+      - name: Set the base sha
+        # Only needed when publishing results.
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        uses: actions/github-script@98814c53be79b1d30f795b907e553d8679345975 # v6.4.0
+        id: base-sha
+        with:
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+            script: |
+                const baseRef = await github.rest.git.getRef({ owner: context.repo.owner, repo: context.repo.repo, ref: 'tags/${{ env.BASE_TAG }}' });
+                return baseRef.data.object.sha;
+
+      - name: Set commit details
+        # Only needed when publishing results.
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        uses: actions/github-script@98814c53be79b1d30f795b907e553d8679345975 # v6.4.0
+        id: commit-timestamp
+        with:
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+            script: |
+                const commit_details = await github.rest.git.getCommit({ owner: context.repo.owner, repo: context.repo.repo, commit_sha: context.sha });
+                return parseInt((new Date( commit_details.data.author.date ).getTime() / 1000).toFixed(0))
+
+      - name: Publish performance results
+        # Only publish results on pushes to trunk.
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }}
+        env:
+            BASE_SHA: ${{ steps.base-sha.outputs.result }}
+            COMMITTED_AT: ${{ steps.commit-timestamp.outputs.result }}
+            CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }}
+            HOST_NAME: "codevitals.run"
+        run: node ./tests/performance/log-results.js $CODEVITALS_PROJECT_TOKEN trunk $GITHUB_SHA $BASE_SHA $COMMITTED_AT $HOST_NAME
+
+      - name: Ensure version-controlled files are not modified or deleted
+        run: git diff --exit-code
+
+  slack-notifications:
+    name: Slack Notifications
+    uses: WordPress/wordpress-develop/.github/workflows/slack-notifications.yml@trunk
+    needs: [ performance ]
+    if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }}
+    with:
+      calling_status: ${{ needs.performance.result == 'success' && 'success' || needs.performance.result == 'cancelled' && 'cancelled' || 'failure' }}
+    secrets:
+      SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }}
+      SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }}
+      SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }}
+      SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }}
+
+  failed-workflow:
+    name: Failed workflow tasks
+    runs-on: ubuntu-latest
+    needs: [ performance, slack-notifications ]
+    if: |
+      always() &&
+      github.repository == 'WordPress/wordpress-develop' &&
+      github.event_name != 'pull_request' &&
+      github.run_attempt < 2 &&
+      (
+        needs.performance.result == 'cancelled' || needs.performance.result == 'failure'
+      )
+
+    steps:
+      - name: Dispatch workflow run
+        uses: actions/github-script@98814c53be79b1d30f795b907e553d8679345975 # v6.4.0
+        with:
+          retries: 2
+          retry-exempt-status-codes: 418
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'failed-workflow.yml',
+              ref: 'trunk',
+              inputs: {
+                run_id: '${{ github.run_id }}'
+              }
+            });

.gitignore

@@ -14,6 +14,7 @@ wp-tests-config.php
 /tests/phpunit/data/plugins/wordpress-importer
 /tests/phpunit/data/.trac-ticket-cache*
 /tests/qunit/compiled.html
+/tests/performance/**/*.test.results.json
 /src/.wp-tests-version
 /node_modules
 /npm-debug.log

package.json

@@ -176,6 +176,7 @@
 		"env:cli": "node ./tools/local-env/scripts/docker.js run cli",
 		"env:logs": "node ./tools/local-env/scripts/docker.js logs",
 		"env:pull": "node ./tools/local-env/scripts/docker.js pull",
+		"test:performance": "node ./tests/performance/run-tests.js",
 		"test:php": "node ./tools/local-env/scripts/docker.js run -T php composer update -W && node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit",
 		"test:e2e": "node ./tests/e2e/run-tests.js",
 		"test:visual": "node ./tests/visual-regression/run-tests.js",

tests/performance/config/bootstrap.js

@@ -0,0 +1,41 @@
+/**
+ * WordPress dependencies.
+ */
+import {
+	clearLocalStorage,
+	enablePageDialogAccept,
+	setBrowserViewport,
+} from '@wordpress/e2e-test-utils';
+
+/**
+ * Timeout, in seconds, that the test should be allowed to run.
+ *
+ * @type {string|undefined}
+ */
+const PUPPETEER_TIMEOUT = process.env.PUPPETEER_TIMEOUT;
+
+// The Jest timeout is increased because these tests are a bit slow.
+jest.setTimeout( PUPPETEER_TIMEOUT || 100000 );
+
+async function setupBrowser() {
+	await clearLocalStorage();
+	await setBrowserViewport( 'large' );
+}
+
+/*
+ * Before every test suite run, delete all content created by the test. This ensures
+ * other posts/comments/etc. aren't dirtying tests and tests don't depend on
+ * each other's side-effects.
+ */
+beforeAll( async () => {
+	enablePageDialogAccept();
+
+	await setBrowserViewport( 'large' );
+	await page.emulateMediaFeatures( [
+		{ name: 'prefers-reduced-motion', value: 'reduce' },
+	] );
+} );
+
+afterEach( async () => {
+	await setupBrowser();
+} );

tests/performance/jest.config.js

@@ -0,0 +1,14 @@
+const config = require( '@wordpress/scripts/config/jest-e2e.config' );
+
+const jestE2EConfig = {
+	...config,
+	setupFilesAfterEnv: [
+		'<rootDir>/config/bootstrap.js',
+	],
+	globals: {
+		// Number of requests to run per test.
+		TEST_RUNS: 20,
+	}
+};
+
+module.exports = jestE2EConfig;

tests/performance/log-results.js

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+
+/**
+ * External dependencies.
+ */
+const fs = require( 'fs' );
+const path = require( 'path' );
+const https = require( 'https' );
+const [ token, branch, hash, baseHash, timestamp, host ] = process.argv.slice( 2 );
+const { median } = require( './utils' );
+
+// The list of test suites to log.
+const testSuites = [
+	'home-block-theme',
+	'home-classic-theme',
+];
+
+// A list of results to parse based on test suites.
+const testResults = testSuites.map(( key ) => ({
+	key,
+	file: `${ key }.test.results.json`,
+}));
+
+// A list of base results to parse based on test suites.
+const baseResults = testSuites.map(( key ) => ({
+	key,
+	file: `base-${ key }.test.results.json`,
+}));
+
+/**
+ * Parse test files into JSON objects.
+ *
+ * @param {string} fileName The name of the file.
+ * @returns An array of parsed objects from each file.
+ */
+const parseFile = ( fileName ) => (
+	JSON.parse(
+		fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
+	)
+);
+
+/**
+ * Gets the array of metrics from a list of results.
+ *
+ * @param {Object[]} results A list of results to format.
+ * @return {Object[]} Metrics.
+ */
+const formatResults = ( results ) => {
+	return results.reduce(
+		( result, { key, file } ) => {
+			return {
+				...result,
+				...Object.fromEntries(
+					Object.entries(
+						parseFile( file ) ?? {}
+					).map( ( [ metric, value ] ) => [
+						key + '-' + metric,
+						median ( value ),
+					] )
+				),
+			};
+		},
+		{}
+	);
+};
+
+const data = new TextEncoder().encode(
+	JSON.stringify( {
+		branch,
+		hash,
+		baseHash,
+		timestamp: parseInt( timestamp, 10 ),
+		metrics: formatResults( testResults ),
+		baseMetrics: formatResults( baseResults ),
+	} )
+);
+
+const options = {
+	hostname: host,
+	port: 443,
+	path: '/api/log?token=' + token,
+	method: 'POST',
+	headers: {
+		'Content-Type': 'application/json',
+		'Content-Length': data.length,
+	},
+};
+
+const req = https.request( options, ( res ) => {
+	console.log( `statusCode: ${ res.statusCode }` );
+
+	res.on( 'data', ( d ) => {
+		process.stdout.write( d );
+	} );
+} );
+
+req.on( 'error', ( error ) => {
+	console.error( error );
+} );
+
+req.write( data );
+req.end();

tests/performance/results.js

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+
+/**
+ * External dependencies.
+ */
+const fs = require( 'fs' );
+const { join } = require( 'path' );
+const { median, getResultsFilename } = require( './utils' );
+
+const testSuites = [
+    'home-classic-theme',
+    'home-block-theme',
+];
+
+console.log( '\n>> 🎉 Results 🎉 \n' );
+
+for ( const testSuite of testSuites ) {
+    const resultsFileName = getResultsFilename( testSuite + '.test' );
+    const resultsPath = join( __dirname, '/specs/', resultsFileName );
+    fs.readFile( resultsPath, "utf8", ( err, data ) => {
+        if ( err ) {
+            console.log( "File read failed:", err );
+            return;
+        }
+        const convertString = testSuite.charAt( 0 ).toUpperCase() + testSuite.slice( 1 );
+        console.log( convertString.replace( /[-]+/g, " " ) + ':' );
+
+        tableData = JSON.parse( data );
+        const rawResults = [];
+
+        for ( var key in tableData ) {
+            if ( tableData.hasOwnProperty( key ) ) {
+                rawResults[ key ] = median( tableData[ key ] );
+            }
+        }
+        console.table( rawResults );
+    });
+}

tests/performance/run-tests.js

@@ -0,0 +1,16 @@
+/**
+ * External dependencies.
+ */
+const dotenv = require( 'dotenv' );
+const dotenv_expand = require( 'dotenv-expand' );
+const { execSync } = require( 'child_process' );
+
+// WP_BASE_URL interpolates LOCAL_PORT, so needs to be parsed by dotenv_expand().
+dotenv_expand.expand( dotenv.config() );
+
+// Run the tests, passing additional arguments through to the test script.
+execSync(
+	'wp-scripts test-e2e --config tests/performance/jest.config.js ' +
+		process.argv.slice( 2 ).join( ' ' ),
+	{ stdio: 'inherit' }
+);

tests/performance/specs/home-block-theme.test.js

@@ -0,0 +1,53 @@
+/**
+ * External dependencies.
+ */
+const { basename, join } = require( 'path' );
+const { writeFileSync } = require( 'fs' );
+const { getResultsFilename } = require( './../utils' );
+
+/**
+ * WordPress dependencies.
+ */
+import { activateTheme, createURL } from '@wordpress/e2e-test-utils';
+
+describe( 'Server Timing - Twenty Twenty Three', () => {
+	const results = {
+		wpBeforeTemplate: [],
+		wpTemplate: [],
+		wpTotal: [],
+	};
+
+	beforeAll( async () => {
+		await activateTheme( 'twentytwentythree' );
+	} );
+
+	afterAll( async () => {
+		const resultsFilename = getResultsFilename( basename( __filename, '.js' ) );
+		writeFileSync(
+			join( __dirname, resultsFilename ),
+			JSON.stringify( results, null, 2 )
+		);
+	} );
+
+	it( 'Server Timing Metrics', async () => {
+		let i = TEST_RUNS;
+		while ( i-- ) {
+			await page.goto( createURL( '/' ) );
+			const navigationTimingJson = await page.evaluate( () =>
+				JSON.stringify( performance.getEntriesByType( 'navigation' ) )
+			);
+
+			const [ navigationTiming ] = JSON.parse( navigationTimingJson );
+
+			results.wpBeforeTemplate.push(
+				navigationTiming.serverTiming[0].duration
+			);
+			results.wpTemplate.push(
+				navigationTiming.serverTiming[1].duration
+			);
+			results.wpTotal.push(
+				navigationTiming.serverTiming[2].duration
+			);
+		}
+	} );
+} );

tests/performance/specs/home-classic-theme.test.js

@@ -0,0 +1,55 @@
+/**
+ * External dependencies.
+ */
+const { basename, join } = require( 'path' );
+const { writeFileSync } = require( 'fs' );
+const { exec } = require( 'child_process' );
+const { getResultsFilename } = require( './../utils' );
+
+/**
+ * WordPress dependencies.
+ */
+import { activateTheme, createURL } from '@wordpress/e2e-test-utils';
+
+describe( 'Server Timing - Twenty Twenty One', () => {
+	const results = {
+		wpBeforeTemplate: [],
+		wpTemplate: [],
+		wpTotal: [],
+	};
+
+	beforeAll( async () => {
+		await activateTheme( 'twentytwentyone' );
+		await exec( 'npm run env:cli -- menu location assign all-pages primary' );
+	} );
+
+	afterAll( async () => {
+		const resultsFilename = getResultsFilename( basename( __filename, '.js' ) );
+		writeFileSync(
+			join( __dirname, resultsFilename ),
+			JSON.stringify( results, null, 2 )
+		);
+	} );
+
+	it( 'Server Timing Metrics', async () => {
+		let i = TEST_RUNS;
+		while ( i-- ) {
+			await page.goto( createURL( '/' ) );
+			const navigationTimingJson = await page.evaluate( () =>
+				JSON.stringify( performance.getEntriesByType( 'navigation' ) )
+			);
+
+			const [ navigationTiming ] = JSON.parse( navigationTimingJson );
+
+			results.wpBeforeTemplate.push(
+				navigationTiming.serverTiming[0].duration
+			);
+			results.wpTemplate.push(
+				navigationTiming.serverTiming[1].duration
+			);
+			results.wpTotal.push(
+				navigationTiming.serverTiming[2].duration
+			);
+		}
+	} );
+} );

tests/performance/utils.js

@@ -0,0 +1,33 @@
+/**
+ * Computes the median number from an array numbers.
+ *
+ * @param {number[]} array
+ *
+ * @return {number} Median.
+ */
+function median( array ) {
+	const mid = Math.floor( array.length / 2 );
+	const numbers = [ ...array ].sort( ( a, b ) => a - b );
+	return array.length % 2 !== 0
+		? numbers[ mid ]
+		: ( numbers[ mid - 1 ] + numbers[ mid ] ) / 2;
+}
+
+/**
+ * Gets the result file name.
+ *
+ * @param {string} File name.
+ *
+ * @return {string} Result file name.
+ */
+function getResultsFilename( fileName ) {
+	const prefixArg = process.argv.find( ( arg ) => arg.startsWith( '--prefix' ) );
+	const fileNamePrefix = prefixArg ? `${prefixArg.split( '=' )[1]}-` : '';
+	const resultsFilename = fileNamePrefix + fileName + '.results.json';
+	return resultsFilename;
+}
+
+module.exports = {
+	median,
+	getResultsFilename,
+};

tests/performance/wp-content/mu-plugins/server-timing.php

@@ -0,0 +1,43 @@
+<?php
+
+add_action(
+	'template_redirect',
+	function() {
+
+		global $timestart;
+
+		$server_timing_values = array();
+		$template_start       = microtime( true );
+
+		$server_timing_values['before-template'] = $template_start - $timestart;
+
+		ob_start();
+
+		add_action(
+			'shutdown',
+			function() use ( $server_timing_values, $template_start ) {
+
+				global $timestart;
+
+				$output = ob_get_clean();
+
+				$server_timing_values['template'] = microtime( true ) - $template_start;
+
+				$server_timing_values['total'] = $server_timing_values['before-template'] + $server_timing_values['template'];
+
+				$header_values = array();
+				foreach ( $server_timing_values as $slug => $value ) {
+					if ( is_float( $value ) ) {
+						$value = round( $value * 1000.0, 2 );
+					}
+					$header_values[] = sprintf( 'wp-%1$s;dur=%2$s', $slug, $value );
+				}
+				header( 'Server-Timing: ' . implode( ', ', $header_values ) );
+
+				echo $output;
+			},
+			PHP_INT_MIN
+		);
+	},
+	PHP_INT_MAX
+);