Build/Test Tools: Make the install testing workflow more flexible.

In [56661], a new GitHub Actions workflow was introduced that focused on running some minimal installation tests for a version of WordPress for every PHP and MySQL combination.

This workflow has tested well, but lacks flexibility and possesses one flaw: tests are only ever performed with currently supported versions, even if the version being tested had a different support policy.

This updates the workflow to be more flexible, allowing all versions of WordPress currently receiving security fixes (back through 4.1) to be tested under the correct support policy.

Additionally, the workflow can now run against the `nightly` build of WordPress. This replaces `latest` as the new default. This allows the tests to be run at any point during a release cycle without the need for an officially tagged version.

Props jorbin, joemcgill, costdev.
See #58977.

git-svn-id: https://develop.svn.wordpress.org/trunk@57218 602fd350-edb4-49c9-b593-d223f7449a82

.github/workflows/install-testing.yml

@@ -1,3 +1,6 @@
+# Confirms that installing WordPress using WP-CLI works successfully.
+#
+# This workflow is not meant to test wordpress-develop checkouts, but rather tagged versions officially available on WordPress.org.
 name: Installation Tests
 
 on:
@@ -11,18 +14,20 @@ on:
     # Always test the workflow when changes are suggested.
     paths:
       - '.github/workflows/install-testing.yml'
+  schedule:
+    - cron: '0 0 * * 1'
   workflow_dispatch:
     inputs:
       wp-version:
         description: 'The version to test installing. Accepts major and minor versions, "latest", or "nightly". Major releases must not end with ".0".'
         type: string
-        default: 'latest'
+        default: 'nightly'
 
 # Cancels all previous workflow runs for pull requests that have not completed.
 concurrency:
   # The concurrency group contains the workflow name and the branch name for pull requests
   # or the commit hash for any other events.
-  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  group: ${{ github.workflow }}-${{ inputs.wp-version || github.event_name == 'pull_request' && github.head_ref || github.sha }}
   cancel-in-progress: true
 
 # Disable permissions for all available scopes by default.
@@ -30,6 +35,60 @@ concurrency:
 permissions: {}
 
 jobs:
+  # Determines the appropriate values for PHP and database versions based on the WordPress version being tested.
+  #
+  # Performs the following steps:
+  # - Checks out the repository.
+  # - Fetches the versions of PHP to test.
+  # - Fetches the versions of MySQL to test.
+  build-matrix:
+    name: Determine PHP Versions to test
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      major-wp-version: ${{ steps.major-wp-version.outputs.version }}
+      php-versions: ${{ steps.php-versions.outputs.versions }}
+      mysql-versions: ${{ steps.mysql-versions.outputs.versions }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        with:
+          show-progress: ${{ runner.debug == '1' && 'true' || 'false' }}
+
+      - name: Determine the major WordPress version
+        id: major-wp-version
+        run: |
+          if [ "${{ inputs.wp-version }}" ] && [ "${{ inputs.wp-version }}" != "nightly" ] && [ "${{ inputs.wp-version }}" != "latest" ]; then
+            echo "version=$(echo "${{ inputs.wp-version }}" | tr '.' '-' | cut -d '-' -f1-2)" >> $GITHUB_OUTPUT
+          elif [ "${{ inputs.wp-version }}" ]; then
+            echo "version=$(echo "${{ inputs.wp-version }}")" >> $GITHUB_OUTPUT
+          else
+            echo "version=nightly" >> $GITHUB_OUTPUT
+          fi
+
+      # Look up the major version's specific PHP support policy when a version is provided.
+      # Otherwise, use the current PHP support policy.
+      - name: Get supported PHP versions
+        id: php-versions
+        run: |
+          if [ "${{ steps.major-wp-version.outputs.version }}" != "latest" ] && [ "${{ steps.major-wp-version.outputs.version }}" != "nightly" ]; then
+            echo "versions=$(jq -r '.["${{ steps.major-wp-version.outputs.version }}"] | @json' .version-support-php.json)" >> $GITHUB_OUTPUT
+          else
+            echo "versions=$(jq -r '.[ (keys[-1]) ] | @json' .version-support-php.json)" >> $GITHUB_OUTPUT
+          fi
+
+      # Look up the major version's specific MySQL support policy when a version is provided.
+      # Otherwise, use the current MySQL support policy.
+      - name: Get supported MySQL versions
+        id: mysql-versions
+        run: |
+          if [ "${{ steps.major-wp-version.outputs.version }}" != "latest" ] && [ "${{ steps.major-wp-version.outputs.version }}" != "nightly" ]; then
+            echo "versions=$(jq -r '.["${{ steps.major-wp-version.outputs.version }}"] | @json' .version-support-mysql.json)" >> $GITHUB_OUTPUT
+          else
+            echo "versions=$(jq -r '.[ (keys[-1]) ] | @json' .version-support-mysql.json)" >> $GITHUB_OUTPUT
+          fi
+
   # Test the WordPress installation process through WP-CLI.
   #
   # Performs the following steps:
@@ -39,26 +98,44 @@ jobs:
   # - Creates a `wp-config.php` file.
   # - Installs WordPress.
   install-tests-mysql:
-    name: WP ${{ inputs.wp-version || 'latest' }} / PHP ${{ matrix.php }} / ${{ 'mariadb' == matrix.db-type && 'MariaDB' || 'MySQL' }} ${{ matrix.db-version }}${{ matrix.multisite && ' multisite' || '' }}
+    name: WP ${{ inputs.wp-version || 'nightly' }} / PHP ${{ matrix.php }} / ${{ 'mariadb' == matrix.db-type && 'MariaDB' || 'MySQL' }} ${{ matrix.db-version }}${{ matrix.multisite && ' multisite' || '' }}
     permissions:
       contents: read
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 10
+    needs: [ build-matrix ]
     strategy:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest ]
-        php: [ '7.0', '7.1', '7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.3' ]
+        php: ${{ fromJSON( needs.build-matrix.outputs.php-versions ) }}
         db-type: [ 'mysql' ]
-        db-version: [ '5.7', '8.0' ]
+        db-version: ${{ fromJSON( needs.build-matrix.outputs.mysql-versions ) }}
         multisite: [ false, true ]
+        memcached: [ false ]
+
+        # Exclude some PHP and MySQL versions that cannot currently be tested with Docker containers.
+        exclude:
+          - php: '5.2'
+          - php: '5.3'
+          - db-version: '5.0'
+          - db-version: '5.1'
+          - db-version: '5.5'
 
     services:
       database:
         image: ${{ matrix.db-type }}:${{ matrix.db-version }}
         ports:
           - 3306
-        options: --health-cmd="mysqladmin ping" --health-interval=30s --health-timeout=10s --health-retries=5 -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=test_db --entrypoint sh ${{ matrix.db-type }}:${{ matrix.db-version }} -c "exec docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=30s
+          --health-timeout=10s
+          --health-retries=5
+          -e MYSQL_ROOT_PASSWORD=root
+          -e MYSQL_DATABASE=test_db
+          --entrypoint sh ${{ matrix.db-type }}:${{ matrix.db-version }}
+          -c "exec docker-entrypoint.sh mysqld${{ matrix.db-version != '5.5' && ' --default-authentication-plugin=mysql_native_password"' || '' }}
 
     steps:
       - name: Set up PHP ${{ matrix.php }}
@@ -66,14 +143,14 @@ jobs:
         with:
           php-version: '${{ matrix.php }}'
           coverage: none
-          tools: wp-cli
+          tools: wp-cli${{ contains( fromJSON('["5.4", "5.5"]'), matrix.php ) && ':2.4.0' || '' }}
 
       - name: Start the database server
         run: |
           sudo systemctl start ${{ matrix.db-type }}
 
       - name: Download WordPress
-        run: wp core download ${{ inputs.wp-version && 'latest' != inputs.wp-version && format( '--version={0}', inputs.wp-version ) || '' }}
+        run: wp core download ${{ inputs.wp-version && format( '--version={0}', inputs.wp-version ) || '--version=nightly' }}
 
       - name: Create wp-config.php file
         run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost=127.0.0.1:${{ job.services.database.ports['3306'] }}

.version-support-mysql.json

@@ -0,0 +1,188 @@
+{
+	"6-5": [
+		"8.2",
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5"
+	],
+	"6-4": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"6-3": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"6-2": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"6-1": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"6-0": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-9": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-8": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-7": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-6": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-5": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-4": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-3": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-2": [
+		"8.0",
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"5-1": [
+		"5.7",
+		"5.6",
+		"5.5"
+	],
+	"5-0": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-9": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-8": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-7": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-6": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-5": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-4": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-3": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-2": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	],
+	"4-1": [
+		"5.7",
+		"5.6",
+		"5.5",
+		"5.1",
+		"5.0"
+	]
+}

.version-support-php.json

@@ -0,0 +1,208 @@
+{
+	"6-3": [
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4",
+		"8.0",
+		"8.1",
+		"8.2",
+		"8.3"
+	],
+	"6-2": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4",
+		"8.0",
+		"8.1",
+		"8.2"
+	],
+	"6-1": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4",
+		"8.0",
+		"8.1",
+		"8.2"
+	],
+	"6-0": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4",
+		"8.0",
+		"8.1"
+	],
+	"5-9": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4",
+		"8.0",
+		"8.1"
+	],
+	"5-8": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4",
+		"8.0"
+	],
+	"5-7": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4",
+		"8.0"
+	],
+	"5-6": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4",
+		"8.0"
+	],
+	"5-5": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4"
+	],
+	"5-4": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4"
+	],
+	"5-3": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3",
+		"7.4"
+	],
+	"5-2": [
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3"
+	],
+	"5-1": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3"
+	],
+	"5-0": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2",
+		"7.3"
+	],
+	"4-9": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6",
+		"7.0",
+		"7.1",
+		"7.2"
+	],
+	"4-8": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6",
+		"7.0",
+		"7.1"
+	],
+	"4-7": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6",
+		"7.0",
+		"7.1"
+	],
+	"4-6": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6",
+		"7.0"
+	],
+	"4-5": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6",
+		"7.0"
+	],
+	"4-4": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6",
+		"7.0"
+	],
+	"4-3": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6"
+	],
+	"4-2": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6"
+	],
+	"4-1": [
+		"5.2",
+		"5.3",
+		"5.4",
+		"5.5",
+		"5.6"
+	]
+}