diff --git a/.github/workflows/sets_check_parallel.yaml b/.github/workflows/sets_check_parallel.yaml new file mode 100644 index 00000000000..1f2ad7b431e --- /dev/null +++ b/.github/workflows/sets_check_parallel.yaml @@ -0,0 +1,19 @@ +name: Sets Check (parallel) + +on: + pull_request: null + push: + branches: + - master + +jobs: + run_all_sets: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: shivammathur/setup-php@v1 + with: + php-version: 7.4 + coverage: none # disable xdebug, pcov + - run: composer install --no-progress + - run: php ci/run_all_sets_parallel.php diff --git a/ci/run_all_sets_parallel.php b/ci/run_all_sets_parallel.php new file mode 100644 index 00000000000..436b7df6431 --- /dev/null +++ b/ci/run_all_sets_parallel.php @@ -0,0 +1,53 @@ +getCpuCores(); +} catch (CouldNotDeterminedCpuCoresException $t) { + echo "WARNING: Could not determine number of CPU cores due to ".$t->getMessage().PHP_EOL; +} + +$tasks = []; +foreach ($setProvider->provide() as $setName) { + if (in_array($setName, $excludedSets, true)) { + continue; + } + $tasks[$setName] = new Task($file, $setName); +} + +if ($taskRunner->run($tasks, $maxProcesses, $sleepSeconds)) { + exit(ShellCode::SUCCESS); +} +exit(ShellCode::ERROR); diff --git a/composer.json b/composer.json index 64055a9cab3..d1cd1d367da 100644 --- a/composer.json +++ b/composer.json @@ -190,6 +190,7 @@ "Rector\\Twig\\Tests\\": "rules/twig/tests", "Rector\\TypeDeclaration\\Tests\\": "rules/type-declaration/tests", "Rector\\Utils\\DocumentationGenerator\\": "utils/documentation-generator/src", + "Rector\\Utils\\ParallelProcessRunner\\": "utils/parallel-process-runner/src", "Rector\\Utils\\PHPStanAttributeTypeSyncer\\": "utils/phpstan-attribute-type-syncer/src", "Rector\\Utils\\PHPStanStaticTypeMapperChecker\\": "utils/phpstan-static-type-mapper-checker/src", "Rector\\ZendToSymfony\\Tests\\": "rules/zend-to-symfony/tests" @@ -241,7 +242,7 @@ "bin/rector check-static-type-mappers", "@check-sets" ], - "check-sets": "php ci/run_all_sets.php", + "check-sets": "php ci/run_all_sets_parallel.php", "check-cs": "vendor/bin/ecs check --ansi", "check-fixtures": "php ci/check_keep_fixtures.php", "check-services": "php ci/check_services_in_yaml_configs.php", diff --git a/packages/node-type-resolver/src/DependencyInjection/PHPStanServicesFactory.php b/packages/node-type-resolver/src/DependencyInjection/PHPStanServicesFactory.php index fdf90dd4806..5e1f683c8d8 100644 --- a/packages/node-type-resolver/src/DependencyInjection/PHPStanServicesFactory.php +++ b/packages/node-type-resolver/src/DependencyInjection/PHPStanServicesFactory.php @@ -54,7 +54,9 @@ final class PHPStanServicesFactory // bleeding edge clean out, see https://github.com/rectorphp/rector/issues/2431 if (Strings::match($phpstanNeonContent, self::BLEEDING_EDGE_PATTERN)) { - $temporaryPhpstanNeon = $currentWorkingDirectory . '/rector-temp-phpstan.neon'; + // Note: We need a unique file per process if rector runs in parallel + $pid = getmypid(); + $temporaryPhpstanNeon = $currentWorkingDirectory . '/rector-temp-phpstan' . $pid . '.neon'; $clearedPhpstanNeonContent = Strings::replace($phpstanNeonContent, self::BLEEDING_EDGE_PATTERN); FileSystem::write($temporaryPhpstanNeon, $clearedPhpstanNeonContent); diff --git a/utils/parallel-process-runner/src/Exception/CouldNotDeterminedCpuCoresException.php b/utils/parallel-process-runner/src/Exception/CouldNotDeterminedCpuCoresException.php new file mode 100644 index 00000000000..c3c34263e87 --- /dev/null +++ b/utils/parallel-process-runner/src/Exception/CouldNotDeterminedCpuCoresException.php @@ -0,0 +1,11 @@ +&1')); + if (! preg_match('#(\d+)#', $str, $matches)) { + throw new CouldNotDeterminedCpuCoresException('wmic failed to get number of cpu cores on windows!'); + } + return (int) $matches[1]; + } + $ret = @shell_exec('nproc'); + if (is_string($ret)) { + $ret = trim($ret); + if (($tmp = filter_var($ret, FILTER_VALIDATE_INT)) !== false) { + return (int) $tmp; + } + } + if (is_readable('/proc/cpuinfo')) { + $cpuinfo = file_get_contents('/proc/cpuinfo'); + $count = substr_count($cpuinfo, 'processor'); + if ($count > 0) { + return $count; + } + } + throw new CouldNotDeterminedCpuCoresException('failed to detect number of CPUs!'); + } +} diff --git a/utils/parallel-process-runner/src/Task.php b/utils/parallel-process-runner/src/Task.php new file mode 100644 index 00000000000..df356b046ad --- /dev/null +++ b/utils/parallel-process-runner/src/Task.php @@ -0,0 +1,34 @@ +pathToFile = $pathToFile; + $this->setName = $setName; + } + + public function getPathToFile(): string + { + return $this->pathToFile; + } + + public function getSetName(): string + { + return $this->setName; + } +} diff --git a/utils/parallel-process-runner/src/TaskRunner.php b/utils/parallel-process-runner/src/TaskRunner.php new file mode 100644 index 00000000000..1a9f2059995 --- /dev/null +++ b/utils/parallel-process-runner/src/TaskRunner.php @@ -0,0 +1,207 @@ +phpExecutable = $phpExecutable; + $this->rectorExecutable = $rectorExecutable; + $this->cwd = $cwd; + $this->failOnlyOnError = $failOnlyOnError; + } + + /** + * @param Task[] $tasks + */ + public function run(array $tasks, int $maxProcesses = 1, int $sleepSeconds = 1): bool + { + $this->printInfo($tasks, $maxProcesses); + + $success = true; + + /** @var Process[] $runningProcesses */ + $runningProcesses = []; + $remainingTasks = $tasks; + $finished = 0; + $total = count($tasks); + do { + $this->sleepIfNecessary($remainingTasks, $runningProcesses, $maxProcesses, $sleepSeconds); + + $this->startProcess($remainingTasks, $runningProcesses, $success, $maxProcesses, $total, $finished); + + $this->evaluateRunningProcesses($runningProcesses, $success, $total, $finished); + + $someProcessesAreStillRunning = count($runningProcesses) > 0; + $notAllProcessesAreStartedYet = count($remainingTasks) > 0; + } while ($someProcessesAreStillRunning || $notAllProcessesAreStartedYet); + + return $success; + } + + /** + * We should sleep when the processes are running in order to not + * exhaust system resources. But we only wanna do this when + * we can't start another processes: + * either because none are left or + * because we reached the threshold of allowed processes + * + * @param string[] $taskIdsToRuns + * @param Process[] $runningProcesses + */ + private function sleepIfNecessary( + array $taskIdsToRuns, + array $runningProcesses, + int $maxProcesses, + int $secondsToSleep + ): void { + $noMoreProcessesAreLeft = count($taskIdsToRuns) === 0; + $maxNumberOfProcessesAreRunning = count($runningProcesses) >= $maxProcesses; + if ($noMoreProcessesAreLeft || $maxNumberOfProcessesAreRunning) { + sleep($secondsToSleep); + } + } + + /** + * @param Task[] $remainingTasks + * @param Task[] $runningProcesses + */ + private function startProcess( + array &$remainingTasks, + array &$runningProcesses, + bool &$success, + int $maxProcesses, + int $total, + int $finished + ): void { + if ($this->canStartAnotherProcess($remainingTasks, $runningProcesses, $maxProcesses)) { + $setName = array_key_first($remainingTasks); + $task = array_shift($remainingTasks); + + try { + $process = $this->createProcess($task); + $process->start(); + $runningProcesses[$setName] = $process; + } catch (Throwable $throwable) { + $success = false; + $this->printError($setName, $throwable->getMessage(), $total, $finished); + } + } + } + + private function evaluateRunningProcesses( + array &$runningProcesses, + bool &$success, + int $total, + int &$finished + ): void { + foreach ($runningProcesses as $setName => $process) { + if (! $process->isRunning()) { + $finished++; + unset($runningProcesses[$setName]); + + try { + $this->evaluateProcess($process); + $this->printSuccess($setName, $total, $finished); + } catch (Throwable $throwable) { + $success = false; + $this->printError( + $setName, + $process->getOutput() . $process->getErrorOutput(), + $total, + $finished + ); + } + } + } + } + + private function canStartAnotherProcess(array $remainingTasks, array $runningProcesses, int $max): bool + { + $hasOpenTasks = count($remainingTasks) > 0; + $moreProcessesCanBeStarted = count($runningProcesses) < $max; + return $hasOpenTasks && $moreProcessesCanBeStarted; + } + + private function createProcess(Task $task): Process + { + $command = [ + $this->phpExecutable, + $this->rectorExecutable, + 'process', + $task->getPathToFile(), + '--set', + $task->getSetName(), + '--dry-run', + ]; + + return new Process($command, $this->cwd); + } + + private function printSuccess(string $set, int $totalTasks, int $finishedTasks): void + { + echo sprintf('(%d/%d) ✔ Set "%s" is OK' . PHP_EOL, $finishedTasks, $totalTasks, $set); + } + + private function printError(string $set, string $output, int $totalTasks, int $finishedTasks): void + { + echo sprintf('(%d/%d) ❌ Set "%s" failed: %s' . PHP_EOL, $finishedTasks, $totalTasks, $set, $output); + } + + private function printInfo(array $tasks, int $maxProcesses): void + { + echo sprintf('Running %d sets with %d parallel processes' . PHP_EOL . PHP_EOL, count($tasks), $maxProcesses); + } + + private function evaluateProcess(Process $process): void + { + if ($process->isSuccessful()) { + return; + } + + // If the process was not successful, there might + // OR an actual error due to an exception + // The latter case is determined via regex + // EITHER be a "possible correction" from rector + + $fullOutput = array_filter([$process->getOutput(), $process->getErrorOutput()]); + + $ouptput = implode("\n", $fullOutput); + + $actualErrorHappened = Strings::match($ouptput, '#(Fatal error)|(\[ERROR\])#'); + + if ($this->failOnlyOnError && ! $actualErrorHappened) { + return; + } + + throw new ProcessResultInvalidException($ouptput); + } +}