[StaticReflection] Add OptimizedDirectorySourceLocator to speedup directory analysis (#5727)

Co-authored-by: kaizen-ci <info@kaizen-ci.org>
This commit is contained in:
Tomas Votruba 2021-03-03 00:42:53 +01:00 committed by GitHub
parent cdd11f69c6
commit 3673e2182c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 222 additions and 156 deletions

View File

@ -22,14 +22,12 @@ jobs:
matrix:
directories:
#- rules
- rules/arguments rules/autodiscovery rules/cakephp rules/carbon rules/code-quality rules/code-quality-strict rules/coding-style rules/composer rules/dead-code rules/dead-doc-block rules/defluent rules/dependency-injection rules/doctrine rules/doctrine-code-quality rules/doctrine-gedmo-to-knplabs rules/downgrade-php70 rules/downgrade-php71 rules/downgrade-php72 rules/downgrade-php73
- rules/arguments rules/autodiscovery rules/cakephp rules/carbon rules/code-quality rules/code-quality-strict rules/coding-style rules/composer rules/dead-code rules/dead-doc-block rules/defluent rules/dependency-injection rules/doctrine rules/doctrine-code-quality rules/doctrine-gedmo-to-knplabs rules/downgrade-php70 rules/downgrade-php71 rules/downgrade-php72 rules/downgrade-php73 rules/naming rules/nette rules/nette-code-quality rules/nette-kdyby
- rules/downgrade-php74 rules/downgrade-php80 rules/early-return rules/generics rules/laravel rules/legacy rules/mockery-to-prophecy rules/mockista-to-mockery rules/mysql-to-mysqli rules/naming rules/nette rules/nette-code-quality rules/nette-kdyby
- rules/downgrade-php74 rules/downgrade-php80 rules/early-return rules/generics rules/laravel rules/legacy rules/mockery-to-prophecy rules/mockista-to-mockery rules/mysql-to-mysqli rules/symfony3 rules/symfony4 rules/symfony5 rules/transform rules/type-declaration rules/visibility
- rules/nette-tester-to-phpunit rules/nette-to-symfony rules/nette-utils-code-quality rules/order rules/php-office rules/php-spec-to-phpunit rules/php52 rules/php53 rules/php54 rules/php55 rules/php56 rules/php70 rules/php71 rules/php72 rules/php73 rules/php74 rules/php80 rules/phpunit rules/phpunit-symfony rules/privatization rules/psr4 rules/removing rules/removing-static rules/renaming rules/restoration rules/sensio rules/symfony rules/symfony-code-quality rules/symfony-php-config rules/symfony2
- rules/symfony3 rules/symfony4 rules/symfony5 rules/transform rules/type-declaration rules/visibility
- packages
- src
- tests
@ -61,7 +59,7 @@ jobs:
- run: composer install --no-progress --ansi
## First run Rector - here can't be --dry-run !!! it would stop the job with it and not commit anything in the future
- run: bin/rector rectify ${{ matrix.directories }} --ansi --no-progress-bar
- run: bin/rector process ${{ matrix.directories }} --ansi --no-progress-bar
- run: vendor/bin/ecs check --match-git-diff --fix --ansi

View File

@ -7,6 +7,7 @@ namespace Rector\NodeTypeResolver\Reflection\BetterReflection\SourceLocatorProvi
use PHPStan\BetterReflection\SourceLocator\Type\AggregateSourceLocator;
use PHPStan\BetterReflection\SourceLocator\Type\SourceLocator;
use PHPStan\Reflection\BetterReflection\SourceLocator\FileNodesFetcher;
use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedDirectorySourceLocator;
use PHPStan\Reflection\BetterReflection\SourceLocator\OptimizedSingleFileSourceLocator;
use Rector\NodeTypeResolver\Contract\SourceLocatorProviderInterface;
use Symplify\SmartFileSystem\SmartFileInfo;
@ -14,9 +15,14 @@ use Symplify\SmartFileSystem\SmartFileInfo;
final class DynamicSourceLocatorProvider implements SourceLocatorProviderInterface
{
/**
* @var SmartFileInfo[]
* @var string[]
*/
private $fileInfos = [];
private $files = [];
/**
* @var array<string, string[]>
*/
private $filesByDirectory = [];
/**
* @var FileNodesFetcher
@ -30,24 +36,36 @@ final class DynamicSourceLocatorProvider implements SourceLocatorProviderInterfa
public function setFileInfo(SmartFileInfo $fileInfo): void
{
$this->fileInfos = [$fileInfo];
$this->files = [$fileInfo->getRealPath()];
}
/**
* @param SmartFileInfo[] $fileInfos
* @param string[] $files
*/
public function addFileInfos(array $fileInfos): void
public function addFiles(array $files): void
{
$this->fileInfos = array_merge($this->fileInfos, $fileInfos);
$this->files = array_merge($this->files, $files);
}
public function provide(): SourceLocator
{
$sourceLocators = [];
foreach ($this->fileInfos as $fileInfo) {
$sourceLocators[] = new OptimizedSingleFileSourceLocator($this->fileNodesFetcher, $fileInfo->getRealPath());
foreach ($this->files as $file) {
$sourceLocators[] = new OptimizedSingleFileSourceLocator($this->fileNodesFetcher, $file);
}
foreach ($this->filesByDirectory as $files) {
$sourceLocators[] = new OptimizedDirectorySourceLocator($this->fileNodesFetcher, $files);
}
return new AggregateSourceLocator($sourceLocators);
}
/**
* @param string[] $files
*/
public function addFilesByDirectory(string $directory, array $files): void
{
$this->filesByDirectory[$directory] = $files;
}
}

View File

@ -178,16 +178,7 @@ abstract class AbstractRectorTestCase extends AbstractKernelTestCase
$this->doTestFileMatchesExpectedContent($inputFileInfo, $expectedFileInfo, $fixtureFileInfo, $extraFileInfos);
$this->originalTempFileInfo = $inputFileInfo;
// runnable?
if (! file_exists($inputFileInfo->getPathname())) {
return;
}
if (! Strings::contains($inputFileInfo->getContents(), RunnableInterface::class)) {
return;
}
$this->assertOriginalAndFixedFileResultEquals($inputFileInfo, $expectedFileInfo);
$this->processRunnableTest($inputFileInfo, $expectedFileInfo);
}
protected function doTestExtraFile(string $expectedExtraFileName, string $expectedExtraContentFilePath): void
@ -345,4 +336,18 @@ abstract class AbstractRectorTestCase extends AbstractKernelTestCase
self::$isInitialized = true;
}
private function processRunnableTest(SmartFileInfo $inputFileInfo, SmartFileInfo $expectedFileInfo): void
{
// runnable?
if (! file_exists($inputFileInfo->getPathname())) {
return;
}
if (! Strings::contains($inputFileInfo->getContents(), RunnableInterface::class)) {
return;
}
$this->assertOriginalAndFixedFileResultEquals($inputFileInfo, $expectedFileInfo);
}
}

View File

@ -99,9 +99,6 @@ CODE_SAMPLE
return null;
}
// dump($expectedBoolName);
// die;
$propertyRename = $this->propertyRenameFactory->createFromExpectedName($node, $expectedBoolName);
if (! $propertyRename instanceof PropertyRename) {
return null;

View File

@ -12,7 +12,8 @@ use Rector\Core\Application\FileSystem\RemovedAndAddedFilesProcessor;
use Rector\Core\Configuration\Configuration;
use Rector\Core\Contract\PostRunnerInterface;
use Rector\Core\Exception\ShouldNotHappenException;
use Rector\NodeTypeResolver\Reflection\BetterReflection\SourceLocatorProvider\DynamicSourceLocatorProvider;
use Rector\Core\FileSystem\PhpFilesFinder;
use Rector\Core\StaticReflection\DynamicSourceLocatorDecorator;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Style\SymfonyStyle;
use Symplify\PackageBuilder\Reflection\PrivatesAccessor;
@ -93,9 +94,14 @@ final class RectorApplication
private $privatesAccessor;
/**
* @var DynamicSourceLocatorProvider
* @var PhpFilesFinder
*/
private $dynamicSourceLocatorProvider;
private $phpFilesFinder;
/**
* @var DynamicSourceLocatorDecorator
*/
private $dynamicSourceLocatorDecorator;
/**
* @param PostRunnerInterface[] $postRunners
@ -109,7 +115,8 @@ final class RectorApplication
RemovedAndAddedFilesProcessor $removedAndAddedFilesProcessor,
SymfonyStyle $symfonyStyle,
PrivatesAccessor $privatesAccessor,
DynamicSourceLocatorProvider $dynamicSourceLocatorProvider,
PhpFilesFinder $phpFilesFinder,
DynamicSourceLocatorDecorator $dynamicSourceLocatorDecorator,
array $postRunners
) {
$this->symfonyStyle = $symfonyStyle;
@ -121,14 +128,16 @@ final class RectorApplication
$this->nodeScopeResolver = $nodeScopeResolver;
$this->privatesAccessor = $privatesAccessor;
$this->postRunners = $postRunners;
$this->dynamicSourceLocatorProvider = $dynamicSourceLocatorProvider;
$this->phpFilesFinder = $phpFilesFinder;
$this->dynamicSourceLocatorDecorator = $dynamicSourceLocatorDecorator;
}
/**
* @param SmartFileInfo[] $phpFileInfos
* @param string[] $paths
*/
public function runOnFileInfos(array $phpFileInfos): void
public function runOnPaths(array $paths): void
{
$phpFileInfos = $this->phpFilesFinder->findInPaths($paths);
$fileCount = count($phpFileInfos);
if ($fileCount === 0) {
return;
@ -139,6 +148,9 @@ final class RectorApplication
// PHPStan has to know about all files!
$this->configurePHPStanNodeScopeResolver($phpFileInfos);
// 0. add files and directories to static locator
$this->dynamicSourceLocatorDecorator->addPaths($paths);
// 1. parse files to nodes
$this->parseFileInfosToNodes($phpFileInfos);
@ -196,7 +208,6 @@ final class RectorApplication
}
$this->nodeScopeResolver->setAnalysedFiles($filePaths);
$this->dynamicSourceLocatorProvider->addFileInfos($fileInfos);
}
/**

View File

@ -4,12 +4,10 @@ declare(strict_types=1);
namespace Rector\Core\Autoloading;
use Nette\Loaders\RobotLoader;
use Rector\Core\Configuration\Option;
use Rector\Core\StaticReflection\DynamicSourceLocatorDecorator;
use Symfony\Component\Console\Input\InputInterface;
use Symplify\PackageBuilder\Parameter\ParameterProvider;
use Symplify\Skipper\SkipCriteriaResolver\SkippedPathsResolver;
use Symplify\SmartFileSystem\FileSystemFilter;
use Symplify\SmartFileSystem\FileSystemGuard;
/**
@ -17,107 +15,50 @@ use Symplify\SmartFileSystem\FileSystemGuard;
*/
final class AdditionalAutoloader
{
/**
* @var string[]
*/
private $autoloadPaths = [];
/**
* @var FileSystemFilter
*/
private $fileSystemFilter;
/**
* @var SkippedPathsResolver
*/
private $skippedPathsResolver;
/**
* @var FileSystemGuard
*/
private $fileSystemGuard;
public function __construct(
FileSystemFilter $fileSystemFilter,
ParameterProvider $parameterProvider,
SkippedPathsResolver $skippedPathsResolver,
FileSystemGuard $fileSystemGuard
) {
$this->autoloadPaths = (array) $parameterProvider->provideParameter(Option::AUTOLOAD_PATHS);
$this->fileSystemFilter = $fileSystemFilter;
$this->skippedPathsResolver = $skippedPathsResolver;
$this->fileSystemGuard = $fileSystemGuard;
}
/**
* @var ParameterProvider
*/
private $parameterProvider;
/**
* @param string[] $source
* @var DynamicSourceLocatorDecorator
*/
public function autoloadWithInputAndSource(InputInterface $input, array $source): void
{
$autoloadDirectories = $this->fileSystemFilter->filterDirectories($this->autoloadPaths);
$autoloadFiles = $this->fileSystemFilter->filterFiles($this->autoloadPaths);
private $dynamicSourceLocatorDecorator;
$this->autoloadFileFromInput($input);
$this->autoloadDirectories($autoloadDirectories);
$this->autoloadFiles($autoloadFiles);
// the scanned file needs to be autoloaded
$directories = $this->fileSystemFilter->filterDirectories($source);
foreach ($directories as $directory) {
// load project autoload
if (file_exists($directory . '/vendor/autoload.php')) {
require_once $directory . '/vendor/autoload.php';
}
}
public function __construct(
FileSystemGuard $fileSystemGuard,
ParameterProvider $parameterProvider,
DynamicSourceLocatorDecorator $dynamicSourceLocatorDecorator
) {
$this->fileSystemGuard = $fileSystemGuard;
$this->parameterProvider = $parameterProvider;
$this->dynamicSourceLocatorDecorator = $dynamicSourceLocatorDecorator;
}
private function autoloadFileFromInput(InputInterface $input): void
public function autoloadWithInputAndSource(InputInterface $input): void
{
if (! $input->hasOption(Option::OPTION_AUTOLOAD_FILE)) {
return;
if ($input->hasOption(Option::OPTION_AUTOLOAD_FILE)) {
$this->autoloadInputAutoloadFile($input);
}
$autoloadPaths = $this->parameterProvider->provideArrayParameter(Option::AUTOLOAD_PATHS);
$this->dynamicSourceLocatorDecorator->addPaths($autoloadPaths);
}
private function autoloadInputAutoloadFile(InputInterface $input): void
{
/** @var string|null $autoloadFile */
$autoloadFile = $input->getOption(Option::OPTION_AUTOLOAD_FILE);
if ($autoloadFile === null) {
return;
}
$this->autoloadFiles([$autoloadFile]);
}
/**
* @param string[] $directories
*/
private function autoloadDirectories(array $directories): void
{
if ($directories === []) {
return;
}
$robotLoader = new RobotLoader();
$robotLoader->ignoreDirs[] = '*Fixtures';
$excludePaths = $this->skippedPathsResolver->resolve();
foreach ($excludePaths as $excludePath) {
$robotLoader->ignoreDirs[] = $excludePath;
}
// last argument is workaround: https://github.com/nette/robot-loader/issues/12
$robotLoader->setTempDirectory(sys_get_temp_dir() . '/_rector_robot_loader');
$robotLoader->addDirectory(...$directories);
$robotLoader->register();
}
/**
* @param string[] $files
*/
private function autoloadFiles(array $files): void
{
foreach ($files as $file) {
$this->fileSystemGuard->ensureFileExists($file, 'Extra autoload');
require_once $file;
}
$this->fileSystemGuard->ensureFileExists($autoloadFile, 'Extra autoload');
$this->dynamicSourceLocatorDecorator->addPaths([$autoloadFile]);
}
}

View File

@ -4,8 +4,6 @@ declare(strict_types=1);
namespace Rector\Core\Console\Command;
use Nette\Utils\Strings;
use Rector\Caching\Application\CachedFileInfoFilterAndReporter;
use Rector\Caching\Detector\ChangedFilesDetector;
use Rector\ChangesReporting\Application\ErrorAndDiffCollector;
use Rector\ChangesReporting\Output\ConsoleOutputFormatter;
@ -16,6 +14,7 @@ use Rector\Core\Configuration\Configuration;
use Rector\Core\Configuration\Option;
use Rector\Core\Console\Output\OutputFormatterCollector;
use Rector\Core\FileSystem\FilesFinder;
use Rector\Core\FileSystem\PhpFilesFinder;
use Rector\Core\Guard\RectorGuard;
use Rector\Core\NonPhpFile\NonPhpFileProcessor;
use Rector\Core\PhpParser\NodeTraverser\RectorNodeTraverser;
@ -27,7 +26,6 @@ use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use Symplify\PackageBuilder\Console\ShellCode;
use Symplify\SmartFileSystem\SmartFileInfo;
final class ProcessCommand extends AbstractCommand
{
@ -86,22 +84,22 @@ final class ProcessCommand extends AbstractCommand
*/
private $symfonyStyle;
/**
* @var CachedFileInfoFilterAndReporter
*/
private $cachedFileInfoFilterAndReporter;
/**
* @var ComposerProcessor
*/
private $composerProcessor;
/**
* @var PhpFilesFinder
*/
private $phpFilesFinder;
public function __construct(
AdditionalAutoloader $additionalAutoloader,
ChangedFilesDetector $changedFilesDetector,
Configuration $configuration,
ErrorAndDiffCollector $errorAndDiffCollector,
FilesFinder $phpFilesFinder,
FilesFinder $filesFinder,
NonPhpFileProcessor $nonPhpFileProcessor,
OutputFormatterCollector $outputFormatterCollector,
RectorApplication $rectorApplication,
@ -109,10 +107,10 @@ final class ProcessCommand extends AbstractCommand
RectorNodeTraverser $rectorNodeTraverser,
StubLoader $stubLoader,
SymfonyStyle $symfonyStyle,
CachedFileInfoFilterAndReporter $cachedFileInfoFilterAndReporter,
ComposerProcessor $composerProcessor
ComposerProcessor $composerProcessor,
PhpFilesFinder $phpFilesFinder
) {
$this->filesFinder = $phpFilesFinder;
$this->filesFinder = $filesFinder;
$this->additionalAutoloader = $additionalAutoloader;
$this->rectorGuard = $rectorGuard;
$this->errorAndDiffCollector = $errorAndDiffCollector;
@ -124,16 +122,14 @@ final class ProcessCommand extends AbstractCommand
$this->nonPhpFileProcessor = $nonPhpFileProcessor;
$this->changedFilesDetector = $changedFilesDetector;
$this->symfonyStyle = $symfonyStyle;
$this->cachedFileInfoFilterAndReporter = $cachedFileInfoFilterAndReporter;
$this->composerProcessor = $composerProcessor;
$this->phpFilesFinder = $phpFilesFinder;
parent::__construct();
}
protected function configure(): void
{
$this->setAliases(['rectify']);
$this->setDescription('Upgrade or refactor source code with provided rectors');
$this->addArgument(
Option::SOURCE,
@ -201,9 +197,9 @@ final class ProcessCommand extends AbstractCommand
$paths = $this->configuration->getPaths();
$phpFileInfos = $this->findPhpFileInfos($paths);
$phpFileInfos = $this->phpFilesFinder->findInPaths($paths);
$this->additionalAutoloader->autoloadWithInputAndSource($input, $paths);
$this->additionalAutoloader->autoloadWithInputAndSource($input);
if ($this->configuration->isCacheDebug()) {
$message = sprintf('[cache] %d files after cache filter', count($phpFileInfos));
@ -212,7 +208,7 @@ final class ProcessCommand extends AbstractCommand
}
$this->configuration->setFileInfos($phpFileInfos);
$this->rectorApplication->runOnFileInfos($phpFileInfos);
$this->rectorApplication->runOnPaths($paths);
// must run after PHP rectors, because they might change class names, and these class names must be changed in configs
$nonPhpFileInfos = $this->filesFinder->findInDirectoriesAndFiles(
@ -250,25 +246,6 @@ final class ProcessCommand extends AbstractCommand
return ShellCode::ERROR;
}
/**
* @param string[] $paths
* @return SmartFileInfo[]
*/
private function findPhpFileInfos(array $paths): array
{
$phpFileInfos = $this->filesFinder->findInDirectoriesAndFiles(
$paths,
$this->configuration->getFileExtensions()
);
// filter out non-PHP php files, e.g. blade templates in Laravel
$phpFileInfos = array_filter($phpFileInfos, function (SmartFileInfo $smartFileInfo): bool {
return ! Strings::endsWith($smartFileInfo->getPathname(), '.blade.php');
});
return $this->cachedFileInfoFilterAndReporter->filterFileInfos($phpFileInfos);
}
private function reportZeroCacheRectorsCondition(): void
{
if (! $this->configuration->isCacheEnabled()) {

View File

@ -0,0 +1,57 @@
<?php
declare(strict_types=1);
namespace Rector\Core\FileSystem;
use Nette\Utils\Strings;
use Rector\Caching\Application\CachedFileInfoFilterAndReporter;
use Rector\Core\Configuration\Configuration;
use Symplify\SmartFileSystem\SmartFileInfo;
final class PhpFilesFinder
{
/**
* @var FilesFinder
*/
private $filesFinder;
/**
* @var Configuration
*/
private $configuration;
/**
* @var CachedFileInfoFilterAndReporter
*/
private $cachedFileInfoFilterAndReporter;
public function __construct(
FilesFinder $filesFinder,
Configuration $configuration,
CachedFileInfoFilterAndReporter $cachedFileInfoFilterAndReporter
) {
$this->filesFinder = $filesFinder;
$this->configuration = $configuration;
$this->cachedFileInfoFilterAndReporter = $cachedFileInfoFilterAndReporter;
}
/**
* @param string[] $paths
* @return SmartFileInfo[]
*/
public function findInPaths(array $paths): array
{
$phpFileInfos = $this->filesFinder->findInDirectoriesAndFiles(
$paths,
$this->configuration->getFileExtensions()
);
// filter out non-PHP php files, e.g. blade templates in Laravel
$phpFileInfos = array_filter($phpFileInfos, function (SmartFileInfo $smartFileInfo): bool {
return ! Strings::endsWith($smartFileInfo->getPathname(), '.blade.php');
});
return $this->cachedFileInfoFilterAndReporter->filterFileInfos($phpFileInfos);
}
}

View File

@ -0,0 +1,62 @@
<?php
declare(strict_types=1);
namespace Rector\Core\StaticReflection;
use Rector\Core\FileSystem\PhpFilesFinder;
use Rector\NodeTypeResolver\Reflection\BetterReflection\SourceLocatorProvider\DynamicSourceLocatorProvider;
use Symplify\SmartFileSystem\FileSystemFilter;
/**
* @see https://phpstan.org/blog/zero-config-analysis-with-static-reflection
* @see https://github.com/rectorphp/rector/issues/3490
*/
final class DynamicSourceLocatorDecorator
{
/**
* @var FileSystemFilter
*/
private $fileSystemFilter;
/**
* @var DynamicSourceLocatorProvider
*/
private $dynamicSourceLocatorProvider;
/**
* @var PhpFilesFinder
*/
private $phpFilesFinder;
public function __construct(
FileSystemFilter $fileSystemFilter,
DynamicSourceLocatorProvider $dynamicSourceLocatorProvider,
PhpFilesFinder $phpFilesFinder
) {
$this->fileSystemFilter = $fileSystemFilter;
$this->dynamicSourceLocatorProvider = $dynamicSourceLocatorProvider;
$this->phpFilesFinder = $phpFilesFinder;
}
/**
* @param string[] $paths
*/
public function addPaths(array $paths): void
{
$files = $this->fileSystemFilter->filterFiles($paths);
$this->dynamicSourceLocatorProvider->addFiles($files);
$directories = $this->fileSystemFilter->filterDirectories($paths);
foreach ($directories as $directory) {
$filesInfosInDirectory = $this->phpFilesFinder->findInPaths([$directory]);
$filesInDirectory = [];
foreach ($filesInfosInDirectory as $fileInfosInDirectory) {
$filesInDirectory[] = $fileInfosInDirectory->getRealPath();
}
$this->dynamicSourceLocatorProvider->addFilesByDirectory($directory, $filesInDirectory);
}
}
}