mirror_php/rector
1
0
Fork 0
mirror of https://github.com/rectorphp/rector.git synced 2025-04-20 23:41:57 +02:00
Code Issues Projects Releases Wiki Activity

Updated Rector to commit 071730611619e0b6eb79d18af29480341d4a0361

Browse Source 
0717306116 Implement a max jobs per worker budget (#4965)
This commit is contained in:
Tomas Votruba 2023-09-10 12:29:23 +00:00
parent 93a246d47b
commit 0208918c02
7 changed files with 43 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.php
View File

@ -12,7 +12,7 @@ return static function (RectorConfig $rectorConfig) : void {
$rectorConfig->skip([]);
$rectorConfig->autoloadPaths([]);
$rectorConfig->bootstrapFiles([]);
$rectorConfig->parallel(120, 16, 20);
$rectorConfig->parallel();
// to avoid autoimporting out of the box
$rectorConfig->importNames(\false, \false);
$rectorConfig->removeUnusedImports(\false);

2
packages/Config/RectorConfig.php
View File

@ -51,7 +51,7 @@ final class RectorConfig extends Container
{
SimpleParameterProvider::setParameter(Option::PARALLEL, \false);
}
public function parallel(int $seconds = 120, int $maxNumberOfProcess = 16, int $jobSize = 20) : void
public function parallel(int $seconds = 120, int $maxNumberOfProcess = 16, int $jobSize = 15) : void
{
SimpleParameterProvider::setParameter(Option::PARALLEL, \true);
SimpleParameterProvider::setParameter(Option::PARALLEL_JOB_TIMEOUT_IN_SECONDS, $seconds);

39
packages/Parallel/Application/ParallelFileProcessor.php
View File

@ -45,6 +45,13 @@ final class ParallelFileProcessor
* @var int
*/
private const SYSTEM_ERROR_LIMIT = 50;
/**
* The number of chunks a worker can process before getting killed.
* In contrast the jobSize defines the maximum size of a chunk, a worker process at a time.
*
* @var int
*/
private const MAX_CHUNKS_PER_WORKER = 8;
/**
* @var \Symplify\EasyParallel\ValueObject\ProcessPool|null
*/
@ -86,8 +93,8 @@ final class ParallelFileProcessor
$this->processPool->quitProcess($processIdentifier);
return;
}
$job = \array_pop($jobs);
$parallelProcess->request([ReactCommand::ACTION => Action::MAIN, Content::FILES => $job]);
$jobsChunk = \array_pop($jobs);
$parallelProcess->request([ReactCommand::ACTION => Action::MAIN, Content::FILES => $jobsChunk]);
});
});
/** @var string $serverAddress */
@ -107,17 +114,15 @@ final class ParallelFileProcessor
\sleep(1);
};
$timeoutInSeconds = SimpleParameterProvider::provideIntParameter(Option::PARALLEL_JOB_TIMEOUT_IN_SECONDS);
for ($i = 0; $i < $numberOfProcesses; ++$i) {
// nothing else to process, stop now
if ($jobs === []) {
break;
}
$fileChunksBudgetPerProcess = [];
$processSpawner = function () use(&$systemErrors, &$fileDiffs, &$jobs, $postFileCallback, &$systemErrorsCount, &$reachedInternalErrorsCountLimit, $mainScript, $input, $serverPort, $streamSelectLoop, $timeoutInSeconds, $handleErrorCallable, &$fileChunksBudgetPerProcess, &$processSpawner) : void {
$processIdentifier = Random::generate();
$workerCommandLine = $this->workerCommandLineFactory->create($mainScript, ProcessCommand::class, 'worker', $input, $processIdentifier, $serverPort);
$fileChunksBudgetPerProcess[$processIdentifier] = self::MAX_CHUNKS_PER_WORKER;
$parallelProcess = new ParallelProcess($workerCommandLine, $streamSelectLoop, $timeoutInSeconds);
$parallelProcess->start(
// 1. callable on data
function (array $json) use($parallelProcess, &$systemErrors, &$fileDiffs, &$jobs, $postFileCallback, &$systemErrorsCount, &$collectedDatas, &$reachedInternalErrorsCountLimit, $processIdentifier) : void {
function (array $json) use($parallelProcess, &$systemErrors, &$fileDiffs, &$jobs, $postFileCallback, &$systemErrorsCount, &$collectedDatas, &$reachedInternalErrorsCountLimit, $processIdentifier, &$fileChunksBudgetPerProcess, &$processSpawner) : void {
// decode arrays to objects
foreach ($json[Bridge::SYSTEM_ERRORS] as $jsonError) {
if (\is_string($jsonError)) {
@ -138,12 +143,19 @@ final class ParallelFileProcessor
$reachedInternalErrorsCountLimit = \true;
$this->processPool->quitAll();
}
if ($fileChunksBudgetPerProcess[$processIdentifier] <= 0) {
// kill the current worker, and spawn a fresh one to free memory
$this->processPool->quitProcess($processIdentifier);
$processSpawner();
return;
}
if ($jobs === []) {
$this->processPool->quitProcess($processIdentifier);
return;
}
$job = \array_pop($jobs);
$parallelProcess->request([ReactCommand::ACTION => Action::MAIN, Content::FILES => $job]);
$jobsChunk = \array_pop($jobs);
$parallelProcess->request([ReactCommand::ACTION => Action::MAIN, Content::FILES => $jobsChunk]);
--$fileChunksBudgetPerProcess[$processIdentifier];
},
// 2. callable on error
$handleErrorCallable,
@ -160,6 +172,13 @@ final class ParallelFileProcessor
}
);
$this->processPool->attachProcess($processIdentifier, $parallelProcess);
};
for ($i = 0; $i < $numberOfProcesses; ++$i) {
// nothing else to process, stop now
if ($jobs === []) {
break;
}
$processSpawner();
}
$streamSelectLoop->run();
if ($reachedSystemErrorsCountLimit) {

4
src/Application/VersionResolver.php
View File

@ -19,12 +19,12 @@ final class VersionResolver
* @api
* @var string
*/
public const PACKAGE_VERSION = '038f8092eaeeaaf044dbd5685d89bc7636143bbc';
public const PACKAGE_VERSION = '071730611619e0b6eb79d18af29480341d4a0361';
/**
* @api
* @var string
*/
public const RELEASE_DATE = '2023-09-10 19:08:00';
public const RELEASE_DATE = '2023-09-10 19:26:44';
/**
* @var int
*/

2
vendor/autoload.php vendored
View File

@ -22,4 +22,4 @@ if (PHP_VERSION_ID < 50600) {
require_once __DIR__ . '/composer/autoload_real.php';
return ComposerAutoloaderInit8e95aa5e7b0ab086df997f016cd618b8::getLoader();
return ComposerAutoloaderInitfe835b1f541b02a5c8bce8f260ecdb75::getLoader();

10
vendor/composer/autoload_real.php vendored
View File

@ -2,7 +2,7 @@
// autoload_real.php @generated by Composer
class ComposerAutoloaderInit8e95aa5e7b0ab086df997f016cd618b8
class ComposerAutoloaderInitfe835b1f541b02a5c8bce8f260ecdb75
{
private static $loader;
@ -22,17 +22,17 @@ class ComposerAutoloaderInit8e95aa5e7b0ab086df997f016cd618b8
return self::$loader;
}
spl_autoload_register(array('ComposerAutoloaderInit8e95aa5e7b0ab086df997f016cd618b8', 'loadClassLoader'), true, true);
spl_autoload_register(array('ComposerAutoloaderInitfe835b1f541b02a5c8bce8f260ecdb75', 'loadClassLoader'), true, true);
self::$loader = $loader = new \Composer\Autoload\ClassLoader(\dirname(__DIR__));
spl_autoload_unregister(array('ComposerAutoloaderInit8e95aa5e7b0ab086df997f016cd618b8', 'loadClassLoader'));
spl_autoload_unregister(array('ComposerAutoloaderInitfe835b1f541b02a5c8bce8f260ecdb75', 'loadClassLoader'));
require __DIR__ . '/autoload_static.php';
call_user_func(\Composer\Autoload\ComposerStaticInit8e95aa5e7b0ab086df997f016cd618b8::getInitializer($loader));
call_user_func(\Composer\Autoload\ComposerStaticInitfe835b1f541b02a5c8bce8f260ecdb75::getInitializer($loader));
$loader->setClassMapAuthoritative(true);
$loader->register(true);
$filesToLoad = \Composer\Autoload\ComposerStaticInit8e95aa5e7b0ab086df997f016cd618b8::$files;
$filesToLoad = \Composer\Autoload\ComposerStaticInitfe835b1f541b02a5c8bce8f260ecdb75::$files;
$requireFile = \Closure::bind(static function ($fileIdentifier, $file) {
if (empty($GLOBALS['__composer_autoload_files'][$fileIdentifier])) {
$GLOBALS['__composer_autoload_files'][$fileIdentifier] = true;

8
vendor/composer/autoload_static.php vendored
View File

@ -4,7 +4,7 @@
namespace Composer\Autoload;
class ComposerStaticInit8e95aa5e7b0ab086df997f016cd618b8
class ComposerStaticInitfe835b1f541b02a5c8bce8f260ecdb75
{
public static $files = array (
'ad155f8f1cf0d418fe49e248db8c661b' => __DIR__ . '/..' . '/react/promise/src/functions_include.php',
@ -2599,9 +2599,9 @@ class ComposerStaticInit8e95aa5e7b0ab086df997f016cd618b8
public static function getInitializer(ClassLoader $loader)
{
return \Closure::bind(function () use ($loader) {
$loader->prefixLengthsPsr4 = ComposerStaticInit8e95aa5e7b0ab086df997f016cd618b8::$prefixLengthsPsr4;
$loader->prefixDirsPsr4 = ComposerStaticInit8e95aa5e7b0ab086df997f016cd618b8::$prefixDirsPsr4;
$loader->classMap = ComposerStaticInit8e95aa5e7b0ab086df997f016cd618b8::$classMap;
$loader->prefixLengthsPsr4 = ComposerStaticInitfe835b1f541b02a5c8bce8f260ecdb75::$prefixLengthsPsr4;
$loader->prefixDirsPsr4 = ComposerStaticInitfe835b1f541b02a5c8bce8f260ecdb75::$prefixDirsPsr4;
$loader->classMap = ComposerStaticInitfe835b1f541b02a5c8bce8f260ecdb75::$classMap;
}, null, ClassLoader::class);
}