Tools: Further automate backporting from Gutenberg to Core

Follow-up for #51491.

Updating WordPress packages is currently a manual process that takes some reading and trial & error to figure out. This PR adds a single npm task called `sync-gutenberg-packages` that automates this entire process.

Props zieladam.
Fixes #55642.



git-svn-id: https://develop.svn.wordpress.org/trunk@53311 602fd350-edb4-49c9-b593-d223f7449a82

Gruntfile.js

@@ -13,7 +13,7 @@ module.exports = function(grunt) {
 		SOURCE_DIR = 'src/',
 		BUILD_DIR = 'build/',
 		WORKING_DIR = grunt.option( 'dev' ) ? SOURCE_DIR : BUILD_DIR,
- 		BANNER_TEXT = '/*! This file is auto-generated */',
+		BANNER_TEXT = '/*! This file is auto-generated */',
 		autoprefixer = require( 'autoprefixer' ),
 		sass = require( 'sass' ),
 		phpUnitWatchGroup = grunt.option( 'group' ),
@@ -80,7 +80,7 @@ module.exports = function(grunt) {
 				]
 			}
 		},
- 		usebanner: {
+		usebanner: {
 			options: {
 				position: 'top',
 				banner: BANNER_TEXT,
@@ -1216,6 +1216,35 @@ module.exports = function(grunt) {
 		'qunit:compiled'
 	] );
 
+	grunt.registerTask( 'sync-gutenberg-packages', function() {
+		if ( grunt.option( 'update-browserlist' ) ) {
+			// Updating the browserlist database is opt-in and up to the release lead.
+			//
+			// Browserlist database should be updated:
+			// * In each release cycle up until RC1
+			// * If Webpack throws a warning about an outdated database
+			//
+			// It should not be updated:
+			// * After the RC1
+			// * When backporting fixes to older WordPress releases.
+			//
+			// For more context, see:
+			// https://github.com/WordPress/wordpress-develop/pull/2621#discussion_r859840515
+			// https://core.trac.wordpress.org/ticket/55559
+			grunt.task.run( 'browserslist:update' );
+		}
+
+		// Install the latest version of the packages already listed in package.json.
+		grunt.task.run( 'wp-packages:update' );
+
+		// Install any new @wordpress packages that are now required.
+		// Update any non-@wordpress deps to the same version as required in the @wordpress packages (e.g. react 16 -> 17).
+		grunt.task.run( 'wp-packages:refresh-deps' );
+
+		// Build the files stored in the src/ directory.
+		grunt.task.run( 'build:dev' );
+	} );
+
 	grunt.renameTask( 'watch', '_watch' );
 
 	grunt.registerTask( 'watch', function() {
@@ -1637,6 +1666,32 @@ module.exports = function(grunt) {
 		} );
 	} );
 
+	grunt.registerTask( 'wp-packages:update', 'Update WordPress packages', function() {
+		const distTag = grunt.option('dist-tag') || 'latest';
+		grunt.log.writeln( `Updating WordPress packages (--dist-tag=${distTag})` );
+		spawn( 'npx', [ 'wp-scripts', 'packages-update', '--', `--dist-tag=${distTag}` ], {
+			cwd: __dirname,
+			stdio: 'inherit',
+		} );
+	} );
+
+	grunt.registerTask( 'browserslist:update', 'Update the local database of browser supports', function() {
+		grunt.log.writeln( `Updating browsers list` );
+		spawn( 'npx', [ 'browserslist@latest', '--update-db' ], {
+			cwd: __dirname,
+			stdio: 'inherit',
+		} );
+	} );
+
+	grunt.registerTask( 'wp-packages:refresh-deps', 'Update version of dependencies in package.json to match the ones listed in the latest WordPress packages', function() {
+		const distTag = grunt.option('dist-tag') || 'latest';
+		grunt.log.writeln( `Updating versions of dependencies listed in package.json (--dist-tag=${distTag})` );
+		spawn( 'node', [ 'tools/release/sync-gutenberg-packages.js', `--dist-tag=${distTag}` ], {
+			cwd: __dirname,
+			stdio: 'inherit',
+		} );
+	} );
+
 	// Patch task.
 	grunt.renameTask('patch_wordpress', 'patch');
 

package.json

@@ -174,6 +174,6 @@
 		"test:php": "node ./tools/local-env/scripts/docker.js run -T php composer update -W && node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit",
 		"test:e2e": "node ./tests/e2e/run-tests.js",
 		"test:visual": "node ./tests/visual-regression/run-tests.js",
-		"wp-packages-update": "wp-scripts packages-update"
+		"sync-gutenberg-packages": "grunt sync-gutenberg-packages"
 	}
 }

tools/release/sync-gutenberg-packages.js

@@ -0,0 +1,227 @@
+/* eslint-disable no-console */
+/**
+ * External dependencies
+ */
+const fs = require( 'fs' );
+const spawn = require( 'cross-spawn' );
+const { zip, uniq, identity, groupBy } = require( 'lodash' );
+
+/**
+ * Constants
+ */
+const WORDPRESS_PACKAGES_PREFIX = '@wordpress/';
+const { getArgFromCLI } = require( `../../node_modules/@wordpress/scripts/utils` );
+const distTag = getArgFromCLI( '--dist-tag' ) || 'latest';
+
+/**
+ * The main function of this task.
+ *
+ * It installs any missing WordPress packages, and updates the
+ * mismatched dependencies versions, e.g. it would detect that Gutenberg
+ * updated react from 16.0.4 to 17.0.2 and install the latter.
+ */
+function main() {
+	const initialPackageJSON = readJSONFile( `package.json` );
+
+	// Install any missing WordPress packages:
+	const missingWordPressPackages = getMissingWordPressPackages();
+	if ( missingWordPressPackages.length ) {
+		console.log( "The following @wordpress dependencies are missing: " );
+		console.log( missingWordPressPackages );
+		console.log( "Installing via npm..." );
+		installPackages( missingWordPressPackages.map( name => [name, distTag] ) );
+	}
+
+	// Update any outdated non-WordPress packages:
+	const versionMismatches = getMismatchedNonWordPressDependencies();
+	if ( versionMismatches.length ) {
+		console.log( "The following dependencies are outdated: " );
+		console.log( versionMismatches );
+		console.log( "Updating via npm..." );
+		const requiredPackages = versionMismatches.map( ( { name, required } ) => [name, required] );
+		installPackages( requiredPackages );
+	}
+
+	const finalPackageJSON = readJSONFile( "package.json" );
+	outputPackageDiffReport(
+		getPackageVersionDiff( initialPackageJSON, finalPackageJSON ),
+	);
+	process.exit( 0 );
+}
+
+/**
+ * @param {string} fileName File to read.
+ * @return {Object} Parsed data.
+ */
+function readJSONFile( fileName ) {
+	const data = fs.readFileSync( fileName, 'utf8' );
+	return JSON.parse( data );
+}
+
+/**
+ * Spawns npm install --save.
+ *
+ * @param {Array} packages List of tuples [packageName, version] to install.
+ * @return {string} CLI output.
+ */
+function installPackages( packages ) {
+	const packagesWithVersion = packages.map(
+		( [packageName, version] ) => `${ packageName }@${ version }`,
+	);
+	return spawn.sync( 'npm', ['install', ...packagesWithVersion, '--save'], {
+		stdio: 'inherit',
+	} );
+}
+
+/**
+ * Computes which @wordpress packages are required by the Gutenberg
+ * dependencies that are missing from WordPress package.json.
+ *
+ * @return {Array} List of tuples [packageName, version].
+ */
+function getMissingWordPressPackages() {
+	const perPackageDeps = getPerPackageDeps();
+	const currentPackages = perPackageDeps.map( ( [name] ) => name );
+
+	const requiredWpPackages = uniq( perPackageDeps
+		// Capture the @wordpress dependencies of our dependencies into a flat list.
+		.flatMap( ( [, dependencies] ) => getWordPressPackages( { dependencies } ) )
+		.sort(),
+	);
+
+	return requiredWpPackages.filter(
+		packageName => !currentPackages.includes( packageName ) );
+}
+
+/**
+ * Computes which third party packages are required by the @wordpress
+ * packages, but not by the WordPress repo itself. This includes
+ * both packages that are missing from package.json and any version
+ * mismatches.
+ *
+ * @return {Array} List of objects {name, required, actual} describing version mismatches.
+ */
+function getMismatchedNonWordPressDependencies() {
+	// Get the installed dependencies from package-lock.json
+	const currentPackageJSON = readJSONFile( "package.json" );
+	const currentPackages = getWordPressPackages( currentPackageJSON );
+
+	const packageLock = readJSONFile( "package-lock.json" );
+	const versionConflicts = Object.entries( packageLock.dependencies )
+		.filter( ( [packageName] ) => currentPackages.includes( packageName ) )
+		.flatMap( ( [, { dependencies }] ) => Object.entries( dependencies || {} ) )
+		.filter( identity )
+		.map( ( [name, { version }] ) => ( {
+			name,
+			required: version,
+			actual: packageLock.dependencies[ name ].version,
+		} ) )
+		.filter( ( { required, actual } ) => required !== actual )
+	;
+
+	// Ensure that all the conflicts can be resolved with the same version
+	const unresolvableConflicts = Object.entries( groupBy( versionConflicts, ( [name] ) => name ) )
+		.map( ( [name, group] ) => [name, group.map( ( [, { required }] ) => required )] )
+		.filter( ( [, group] ) => group.length > 1 );
+	if ( unresolvableConflicts.length > 0 ) {
+		console.error( "Can't resolve some conflicts automatically." );
+		console.error( "Multiple required versions of the following packages were detected:" );
+		console.error( unresolvableConflicts );
+		process.exit( 1 );
+	}
+	return versionConflicts;
+}
+
+/**
+ * Returns a list of dependencies of each @wordpress dependency.
+ *
+ * @return {Object} An object of shape {packageName: [[packageName, version]]}.
+ */
+function getPerPackageDeps() {
+	// Get the dependencies currently listed in the wordpress-develop package.json
+	const currentPackageJSON = readJSONFile( "package.json" );
+	const currentPackages = getWordPressPackages( currentPackageJSON );
+
+	// Get the dependencies that the above dependencies list in their package.json.
+	const deps = currentPackages
+		.map( ( packageName ) => `node_modules/${ packageName }/package.json` )
+		.map( ( jsonPath ) => readJSONFile( jsonPath ).dependencies );
+	return zip( currentPackages, deps );
+}
+
+/**
+ * Takes unserialized package.json data and returns a list of @wordpress dependencies.
+ *
+ * @param {Object} dependencies unserialized package.json data.
+ * @return {string[]} a list of @wordpress dependencies.
+ */
+function getWordPressPackages( { dependencies = {} } ) {
+	return Object.keys( dependencies )
+		.filter( isWordPressPackage );
+}
+
+/**
+ * Returns true if packageName represents a @wordpress package.
+ *
+ * @param {string} packageName Package name to test.
+ * @return {boolean} Is it a @wodpress package?
+ */
+function isWordPressPackage( packageName ) {
+	return packageName.startsWith( WORDPRESS_PACKAGES_PREFIX );
+}
+
+/**
+ * Computes the dependencies difference between two unserialized
+ * package JSON objects. Needed only for the final reporting.
+ *
+ * @param {Object} initialPackageJSON Initial package JSON data.
+ * @param {Object} finalPackageJSON Final package JSON data.
+ * @return {Object} Delta.
+ */
+function getPackageVersionDiff( initialPackageJSON, finalPackageJSON ) {
+	const diff = ['dependencies', 'devDependencies'].reduce(
+		( result, keyPackageJSON ) => {
+			return Object.keys(
+				finalPackageJSON[ keyPackageJSON ] || {},
+			).reduce( ( _result, dependency ) => {
+				const initial =
+					initialPackageJSON[ keyPackageJSON ][ dependency ];
+				const final = finalPackageJSON[ keyPackageJSON ][ dependency ];
+				if ( initial !== final ) {
+					_result.push( { dependency, initial, final } );
+				}
+				return _result;
+			}, result );
+		},
+		[],
+	);
+	return diff.sort( ( a, b ) => a.dependency.localeCompare( b.dependency ) );
+}
+
+/**
+ * Prints the delta between two package.json files.
+ *
+ * @param {Object} packageDiff Delta.
+ */
+function outputPackageDiffReport( packageDiff ) {
+	const readableDiff =
+		packageDiff
+			.map( ( { dependency, initial, final } ) => {
+				return `${ dependency }: ${ initial } -> ${ final }`;
+			} )
+			.filter( identity );
+	if ( !readableDiff.length ) {
+		console.log( 'No changes detected' );
+		return;
+	}
+	console.log(
+		[
+			'The following package versions were changed:',
+			...readableDiff,
+		].join( '\n' ),
+	);
+}
+
+main();
+
+/* eslint-enable no-console */