Add parallel execution to ci/run_all_sets

.github/workflows/sets_check_parallel.yaml

@@ -0,0 +1,19 @@
+name: Sets Check (parallel)
+
+on:
+    pull_request: null
+    push:
+        branches:
+            - master
+
+jobs:
+    run_all_sets:
+        runs-on: ubuntu-latest
+        steps:
+            -   uses: actions/checkout@v2
+            -   uses: shivammathur/setup-php@v1
+                with:
+                    php-version: 7.4
+                    coverage: none # disable xdebug, pcov
+            -   run: composer install --no-progress
+            -   run: php ci/run_all_sets_parallel.php

ci/run_all_sets_parallel.php

@@ -0,0 +1,53 @@
+<?php
+
+declare(strict_types=1);
+
+use Rector\Core\Set\SetProvider;
+use Rector\Utils\ParallelProcessRunner\Exception\CouldNotDeterminedCpuCoresException;
+use Rector\Utils\ParallelProcessRunner\SystemInfo;
+use Rector\Utils\ParallelProcessRunner\Task;
+use Rector\Utils\ParallelProcessRunner\TaskRunner;
+use Symplify\PackageBuilder\Console\ShellCode;
+
+require __DIR__.'/../vendor/autoload.php';
+
+$setProvider = new SetProvider();
+
+// We'll only check one file for now.
+// This makes sure that all sets are "runnable" but keeps the runtime at a managable level
+$file         = __DIR__.'/../src/Rector/AbstractRector.php';
+
+$excludedSets = [
+    // required Kernel class to be set in parameters
+    'symfony-code-quality',
+];
+
+$cwd = __DIR__."/..";
+
+$systemInfo   = new SystemInfo();
+$taskRunner   = new TaskRunner(
+    "php",
+    $cwd."/bin/rector",
+    $cwd,
+    true
+);
+$sleepSeconds = 1;
+$maxProcesses = 1;
+try {
+    $maxProcesses = $systemInfo->getCpuCores();
+} catch (CouldNotDeterminedCpuCoresException $t) {
+    echo "WARNING: Could not determine number of CPU cores due to ".$t->getMessage().PHP_EOL;
+}
+
+$tasks = [];
+foreach ($setProvider->provide() as $setName) {
+    if (in_array($setName, $excludedSets, true)) {
+        continue;
+    }
+    $tasks[$setName] = new Task($file, $setName);
+}
+
+if ($taskRunner->run($tasks, $maxProcesses, $sleepSeconds)) {
+    exit(ShellCode::SUCCESS);
+}
+exit(ShellCode::ERROR);

composer.json

@@ -190,6 +190,7 @@
             "Rector\\Twig\\Tests\\": "rules/twig/tests",
             "Rector\\TypeDeclaration\\Tests\\": "rules/type-declaration/tests",
             "Rector\\Utils\\DocumentationGenerator\\": "utils/documentation-generator/src",
+            "Rector\\Utils\\ParallelProcessRunner\\": "utils/parallel-process-runner/src",
             "Rector\\Utils\\PHPStanAttributeTypeSyncer\\": "utils/phpstan-attribute-type-syncer/src",
             "Rector\\Utils\\PHPStanStaticTypeMapperChecker\\": "utils/phpstan-static-type-mapper-checker/src",
             "Rector\\ZendToSymfony\\Tests\\": "rules/zend-to-symfony/tests"
@@ -241,7 +242,7 @@
             "bin/rector check-static-type-mappers",
             "@check-sets"
         ],
-        "check-sets": "php ci/run_all_sets.php",
+        "check-sets": "php ci/run_all_sets_parallel.php",
         "check-cs": "vendor/bin/ecs check --ansi",
         "check-fixtures": "php ci/check_keep_fixtures.php",
         "check-services": "php ci/check_services_in_yaml_configs.php",

packages/node-type-resolver/src/DependencyInjection/PHPStanServicesFactory.php

@@ -54,7 +54,9 @@ final class PHPStanServicesFactory
 
             // bleeding edge clean out, see https://github.com/rectorphp/rector/issues/2431
             if (Strings::match($phpstanNeonContent, self::BLEEDING_EDGE_PATTERN)) {
-                $temporaryPhpstanNeon = $currentWorkingDirectory . '/rector-temp-phpstan.neon';
+                // Note: We need a unique file per process if rector runs in parallel
+                $pid = getmypid();
+                $temporaryPhpstanNeon = $currentWorkingDirectory . '/rector-temp-phpstan' . $pid . '.neon';
                 $clearedPhpstanNeonContent = Strings::replace($phpstanNeonContent, self::BLEEDING_EDGE_PATTERN);
                 FileSystem::write($temporaryPhpstanNeon, $clearedPhpstanNeonContent);
 

utils/parallel-process-runner/src/Exception/CouldNotDeterminedCpuCoresException.php

@@ -0,0 +1,11 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\Utils\ParallelProcessRunner\Exception;
+
+use RuntimeException;
+
+class CouldNotDeterminedCpuCoresException extends RuntimeException
+{
+}

utils/parallel-process-runner/src/Exception/ProcessResultInvalidException.php

@@ -0,0 +1,11 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\Utils\ParallelProcessRunner\Exception;
+
+use RuntimeException;
+
+class ProcessResultInvalidException extends RuntimeException
+{
+}

utils/parallel-process-runner/src/SystemInfo.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\Utils\ParallelProcessRunner;
+
+use Rector\Utils\ParallelProcessRunner\Exception\CouldNotDeterminedCpuCoresException;
+
+final class SystemInfo
+{
+    /**
+     * @see https://gist.github.com/divinity76/01ef9ca99c111565a72d3a8a6e42f7fb
+     *
+     * returns number of cpu cores
+     * Copyleft 2018, license: WTFPL
+     */
+    public function getCpuCores(): int
+    {
+        if (defined('PHP_WINDOWS_VERSION_MAJOR')) {
+            $str = trim(shell_exec('wmic cpu get NumberOfCores 2>&1'));
+            if (! preg_match('#(\d+)#', $str, $matches)) {
+                throw new CouldNotDeterminedCpuCoresException('wmic failed to get number of cpu cores on windows!');
+            }
+            return (int) $matches[1];
+        }
+        $ret = @shell_exec('nproc');
+        if (is_string($ret)) {
+            $ret = trim($ret);
+            if (($tmp = filter_var($ret, FILTER_VALIDATE_INT)) !== false) {
+                return (int) $tmp;
+            }
+        }
+        if (is_readable('/proc/cpuinfo')) {
+            $cpuinfo = file_get_contents('/proc/cpuinfo');
+            $count = substr_count($cpuinfo, 'processor');
+            if ($count > 0) {
+                return $count;
+            }
+        }
+        throw new CouldNotDeterminedCpuCoresException('failed to detect number of CPUs!');
+    }
+}

utils/parallel-process-runner/src/Task.php

@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\Utils\ParallelProcessRunner;
+
+final class Task
+{
+    /**
+     * @var string
+     */
+    private $pathToFile;
+
+    /**
+     * @var string
+     */
+    private $setName;
+
+    public function __construct(string $pathToFile, string $setName)
+    {
+        $this->pathToFile = $pathToFile;
+        $this->setName = $setName;
+    }
+
+    public function getPathToFile(): string
+    {
+        return $this->pathToFile;
+    }
+
+    public function getSetName(): string
+    {
+        return $this->setName;
+    }
+}

utils/parallel-process-runner/src/TaskRunner.php

@@ -0,0 +1,207 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rector\Utils\ParallelProcessRunner;
+
+use Nette\Utils\Strings;
+use Rector\Utils\ParallelProcessRunner\Exception\ProcessResultInvalidException;
+use Symfony\Component\Process\Process;
+use Throwable;
+
+final class TaskRunner
+{
+    /**
+     * @var string
+     */
+    private $phpExecutable;
+
+    /**
+     * @var string
+     */
+    private $rectorExecutable;
+
+    /**
+     * @var string
+     */
+    private $cwd;
+
+    /**
+     * @var bool
+     */
+    private $failOnlyOnError = false;
+
+    public function __construct(string $phpExecutable, string $rectorExecutable, string $cwd, bool $failOnlyOnError)
+    {
+        $this->phpExecutable = $phpExecutable;
+        $this->rectorExecutable = $rectorExecutable;
+        $this->cwd = $cwd;
+        $this->failOnlyOnError = $failOnlyOnError;
+    }
+
+    /**
+     * @param Task[] $tasks
+     */
+    public function run(array $tasks, int $maxProcesses = 1, int $sleepSeconds = 1): bool
+    {
+        $this->printInfo($tasks, $maxProcesses);
+
+        $success = true;
+
+        /** @var Process[] $runningProcesses */
+        $runningProcesses = [];
+        $remainingTasks = $tasks;
+        $finished = 0;
+        $total = count($tasks);
+        do {
+            $this->sleepIfNecessary($remainingTasks, $runningProcesses, $maxProcesses, $sleepSeconds);
+
+            $this->startProcess($remainingTasks, $runningProcesses, $success, $maxProcesses, $total, $finished);
+
+            $this->evaluateRunningProcesses($runningProcesses, $success, $total, $finished);
+
+            $someProcessesAreStillRunning = count($runningProcesses) > 0;
+            $notAllProcessesAreStartedYet = count($remainingTasks) > 0;
+        } while ($someProcessesAreStillRunning || $notAllProcessesAreStartedYet);
+
+        return $success;
+    }
+
+    /**
+     * We should sleep when the processes are running in order to not
+     * exhaust system resources. But we only wanna do this when
+     * we can't start another processes:
+     * either because none are left or
+     * because we reached the threshold of allowed processes
+     *
+     * @param string[] $taskIdsToRuns
+     * @param Process[] $runningProcesses
+     */
+    private function sleepIfNecessary(
+        array $taskIdsToRuns,
+        array $runningProcesses,
+        int $maxProcesses,
+        int $secondsToSleep
+    ): void {
+        $noMoreProcessesAreLeft = count($taskIdsToRuns) === 0;
+        $maxNumberOfProcessesAreRunning = count($runningProcesses) >= $maxProcesses;
+        if ($noMoreProcessesAreLeft || $maxNumberOfProcessesAreRunning) {
+            sleep($secondsToSleep);
+        }
+    }
+
+    /**
+     * @param Task[] $remainingTasks
+     * @param Task[] $runningProcesses
+     */
+    private function startProcess(
+        array &$remainingTasks,
+        array &$runningProcesses,
+        bool &$success,
+        int $maxProcesses,
+        int $total,
+        int $finished
+    ): void {
+        if ($this->canStartAnotherProcess($remainingTasks, $runningProcesses, $maxProcesses)) {
+            $setName = array_key_first($remainingTasks);
+            $task = array_shift($remainingTasks);
+
+            try {
+                $process = $this->createProcess($task);
+                $process->start();
+                $runningProcesses[$setName] = $process;
+            } catch (Throwable $throwable) {
+                $success = false;
+                $this->printError($setName, $throwable->getMessage(), $total, $finished);
+            }
+        }
+    }
+
+    private function evaluateRunningProcesses(
+        array &$runningProcesses,
+        bool &$success,
+        int $total,
+        int &$finished
+    ): void {
+        foreach ($runningProcesses as $setName => $process) {
+            if (! $process->isRunning()) {
+                $finished++;
+                unset($runningProcesses[$setName]);
+
+                try {
+                    $this->evaluateProcess($process);
+                    $this->printSuccess($setName, $total, $finished);
+                } catch (Throwable $throwable) {
+                    $success = false;
+                    $this->printError(
+                        $setName,
+                        $process->getOutput() . $process->getErrorOutput(),
+                        $total,
+                        $finished
+                    );
+                }
+            }
+        }
+    }
+
+    private function canStartAnotherProcess(array $remainingTasks, array $runningProcesses, int $max): bool
+    {
+        $hasOpenTasks = count($remainingTasks) > 0;
+        $moreProcessesCanBeStarted = count($runningProcesses) < $max;
+        return $hasOpenTasks && $moreProcessesCanBeStarted;
+    }
+
+    private function createProcess(Task $task): Process
+    {
+        $command = [
+            $this->phpExecutable,
+            $this->rectorExecutable,
+            'process',
+            $task->getPathToFile(),
+            '--set',
+            $task->getSetName(),
+            '--dry-run',
+        ];
+
+        return new Process($command, $this->cwd);
+    }
+
+    private function printSuccess(string $set, int $totalTasks, int $finishedTasks): void
+    {
+        echo sprintf('(%d/%d) ✔ Set "%s" is OK' . PHP_EOL, $finishedTasks, $totalTasks, $set);
+    }
+
+    private function printError(string $set, string $output, int $totalTasks, int $finishedTasks): void
+    {
+        echo sprintf('(%d/%d) ❌ Set "%s" failed: %s' . PHP_EOL, $finishedTasks, $totalTasks, $set, $output);
+    }
+
+    private function printInfo(array $tasks, int $maxProcesses): void
+    {
+        echo sprintf('Running %d sets with %d parallel processes' . PHP_EOL . PHP_EOL, count($tasks), $maxProcesses);
+    }
+
+    private function evaluateProcess(Process $process): void
+    {
+        if ($process->isSuccessful()) {
+            return;
+        }
+
+        // If the process was not successful, there might
+        // OR an actual error due to an exception
+        // The latter case is determined via regex
+        // EITHER be a "possible correction" from rector
+
+        $fullOutput = array_filter([$process->getOutput(), $process->getErrorOutput()]);
+
+        $ouptput = implode("\n", $fullOutput);
+
+        $actualErrorHappened = Strings::match($ouptput, '#(Fatal error)|(\[ERROR\])#');
+
+        if ($this->failOnlyOnError && ! $actualErrorHappened) {
+            return;
+        }
+
+        throw new ProcessResultInvalidException($ouptput);
+    }
+}