mirror_php/PHP-Parser
1
0
Fork 0
mirror of https://github.com/nikic/PHP-Parser.git synced 2025-04-21 06:22:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add reverse emulation support

Browse Source
This commit is contained in:
Nikita Popov 2020-08-01 22:34:36 +02:00
parent 3698f4b727
commit 1a1300aa2a
9 changed files with 101 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
lib/PhpParser/Lexer.php
View File

@ -34,23 +34,8 @@ class Lexer
* first three. For more info see getNextToken() docs.
*/
public function __construct(array $options = []) {
// Compatibility define for PHP < 7.4.
if (!defined('T_BAD_CHARACTER')) {
\define('T_BAD_CHARACTER', -1);
}
// Compatibility defines for PHP < 8.0.
if (!defined('T_NAME_QUALIFIED')) {
\define('T_NAME_QUALIFIED', -2);
}
if (!defined('T_NAME_FULLY_QUALIFIED')) {
\define('T_NAME_FULLY_QUALIFIED', -3);
}
if (!defined('T_NAME_RELATIVE')) {
\define('T_NAME_RELATIVE', -4);
}
// Create Map from internal tokens to PhpParser tokens.
$this->defineCompatibilityTokens();
$this->tokenMap = $this->createTokenMap();
// map of tokens to drop while lexing (the map is only used for isset lookup,
@ -418,6 +403,33 @@ class Lexer
return substr($textAfter, strlen($matches[0]));
}
private function defineCompatibilityTokens() {
// PHP 7.4
if (!defined('T_BAD_CHARACTER')) {
\define('T_BAD_CHARACTER', -1);
}
if (!defined('T_FN')) {
\define('T_FN', -2);
}
if (!defined('T_COALESCE_EQUAL')) {
\define('T_COALESCE_EQUAL', -3);
}
// PHP 8.0
if (!defined('T_NAME_QUALIFIED')) {
\define('T_NAME_QUALIFIED', -4);
}
if (!defined('T_NAME_FULLY_QUALIFIED')) {
\define('T_NAME_FULLY_QUALIFIED', -5);
}
if (!defined('T_NAME_RELATIVE')) {
\define('T_NAME_RELATIVE', -6);
}
if (!defined('T_MATCH')) {
\define('T_MATCH', -7);
}
}
/**
* Creates the token map.
*
@ -463,9 +475,12 @@ class Lexer
}
// Assign tokens for which we define compatibility constants, as token_name() does not know them.
$tokenMap[\T_FN] = Tokens::T_FN;
$tokenMap[\T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
$tokenMap[\T_NAME_QUALIFIED] = Tokens::T_NAME_QUALIFIED;
$tokenMap[\T_NAME_FULLY_QUALIFIED] = Tokens::T_NAME_FULLY_QUALIFIED;
$tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
$tokenMap[\T_MATCH] = Tokens::T_MATCH;
return $tokenMap;
}

19
lib/PhpParser/Lexer/Emulative.php
View File

@ -14,13 +14,9 @@ use PhpParser\Parser\Tokens;
class Emulative extends Lexer
{
const PHP_7_3 = '7.3.0dev';
const PHP_7_4 = '7.4.0dev';
const PHP_8_0 = '8.0.0dev';
const T_COALESCE_EQUAL = 1007;
const T_FN = 1008;
const T_MATCH = 1009;
const PHP_7_3 = '7.3dev';
const PHP_7_4 = '7.4dev';
const PHP_8_0 = '8.0dev';
const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
@ -53,10 +49,6 @@ REGEX;
$this->tokenEmulators[] = new MatchTokenEmulator();
$this->tokenEmulators[] = new CoaleseEqualTokenEmulator();
$this->tokenEmulators[] = new NumericLiteralSeparatorEmulator();
$this->tokenMap[self::T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
$this->tokenMap[self::T_FN] = Tokens::T_FN;
$this->tokenMap[self::T_MATCH] = Tokens::T_MATCH;
}
public function startLexing(string $code, ErrorHandler $errorHandler = null) {
@ -83,13 +75,16 @@ REGEX;
}
}
// add token emulation
foreach ($this->tokenEmulators as $tokenEmulator) {
$emulatorPhpVersion = $tokenEmulator->getPhpVersion();
if (version_compare(\PHP_VERSION, $emulatorPhpVersion, '<')
&& version_compare($this->targetPhpVersion, $emulatorPhpVersion, '>=')
&& $tokenEmulator->isEmulationNeeded($code)) {
$this->tokens = $tokenEmulator->emulate($code, $this->tokens);
} else if (version_compare(\PHP_VERSION, $emulatorPhpVersion, '>=')
&& version_compare($this->targetPhpVersion, $emulatorPhpVersion, '<')
&& $tokenEmulator->isEmulationNeeded($code)) {
$this->tokens = $tokenEmulator->reverseEmulate($code, $this->tokens);
}
}
}

6
lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php
View File

@ -38,4 +38,10 @@ final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
return $tokens;
}
public function reverseEmulate(string $code, array $tokens): array
{
// ??= was not valid code previously, don't bother.
return $tokens;
}
}

2
lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php
View File

@ -18,6 +18,6 @@ final class FnTokenEmulator extends KeywordEmulator
public function getKeywordToken(): int
{
return Emulative::T_FN;
return \T_FN;
}
}

16
lib/PhpParser/Lexer/TokenEmulator/KeywordEmulator.php
View File

@ -2,8 +2,6 @@
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\Lexer\Emulative;
abstract class KeywordEmulator implements TokenEmulatorInterface
{
abstract function getKeywordString(): string;
@ -20,7 +18,7 @@ abstract class KeywordEmulator implements TokenEmulatorInterface
foreach ($tokens as $i => $token) {
if ($token[0] === T_STRING && strtolower($token[1]) === $keywordString) {
$previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $i);
if ($previousNonSpaceToken !== null && $previousNonSpaceToken[0] === T_OBJECT_OPERATOR) {
if ($previousNonSpaceToken !== null && $previousNonSpaceToken[0] === \T_OBJECT_OPERATOR) {
continue;
}
@ -47,4 +45,16 @@ abstract class KeywordEmulator implements TokenEmulatorInterface
return null;
}
public function reverseEmulate(string $code, array $tokens): array
{
$keywordToken = $this->getKeywordToken();
foreach ($tokens as $i => $token) {
if ($token[0] === $keywordToken) {
$tokens[$i][0] = \T_STRING;
}
}
return $tokens;
}
}

2
lib/PhpParser/Lexer/TokenEmulator/MatchTokenEmulator.php
View File

@ -18,6 +18,6 @@ final class MatchTokenEmulator extends KeywordEmulator
public function getKeywordToken(): int
{
return Emulative::T_MATCH;
return \T_MATCH;
}
}

6
lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php
View File

@ -96,4 +96,10 @@ final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
return is_float($num) ? T_DNUMBER : T_LNUMBER;
}
public function reverseEmulate(string $code, array $tokens): array
{
// Numeric separators were not legal code previously, don't bother.
return $tokens;
}
}

5
lib/PhpParser/Lexer/TokenEmulator/TokenEmulatorInterface.php
View File

@ -13,4 +13,9 @@ interface TokenEmulatorInterface
* @return array Modified Tokens
*/
public function emulate(string $code, array $tokens): array;
/**
* @return array Modified Tokens
*/
public function reverseEmulate(string $code, array $tokens): array;
}

37
test/PhpParser/Lexer/EmulativeTest.php
View File

@ -3,6 +3,7 @@
namespace PhpParser\Lexer;
use PhpParser\ErrorHandler;
use PhpParser\Lexer;
use PhpParser\LexerTest;
use PhpParser\Parser\Tokens;
@ -49,6 +50,9 @@ class EmulativeTest extends LexerTest
public function provideTestReplaceKeywords() {
return [
// PHP 8.0
['match', Tokens::T_MATCH],
// PHP 7.4
['fn', Tokens::T_FN],
@ -70,18 +74,21 @@ class EmulativeTest extends LexerTest
];
}
private function assertSameTokens(array $expectedTokens, Lexer $lexer) {
$tokens = [];
while (0 !== $token = $lexer->getNextToken($text)) {
$tokens[] = [$token, $text];
}
$this->assertSame($expectedTokens, $tokens);
}
/**
* @dataProvider provideTestLexNewFeatures
*/
public function testLexNewFeatures($code, array $expectedTokens) {
$lexer = $this->getLexer();
$lexer->startLexing('<?php ' . $code);
$tokens = [];
while (0 !== $token = $lexer->getNextToken($text)) {
$tokens[] = [$token, $text];
}
$this->assertSame($expectedTokens, $tokens);
$this->assertSameTokens($expectedTokens, $lexer);
}
/**
@ -255,4 +262,22 @@ class EmulativeTest extends LexerTest
]],
];
}
/**
* @dataProvider provideTestTargetVersion
*/
public function testTargetVersion(string $phpVersion, string $code, array $expectedTokens) {
$lexer = $this->getLexer(['phpVersion' => $phpVersion]);
$lexer->startLexing('<?php ' . $code);
$this->assertSameTokens($expectedTokens, $lexer);
}
public function provideTestTargetVersion() {
return [
['8.0', 'match', [[Tokens::T_MATCH, 'match']]],
['7.4', 'match', [[Tokens::T_STRING, 'match']]],
['7.4', 'fn', [[Tokens::T_FN, 'fn']]],
['7.3', 'fn', [[Tokens::T_STRING, 'fn']]],
];
}
}