mirror of
https://github.com/nikic/PHP-Parser.git
synced 2025-01-17 07:08:14 +01:00
[PHP 7.4] Add support for numeric literal separators (#615)
Implements RFC https://wiki.php.net/rfc/numeric_literal_separator. Closes #614.
This commit is contained in:
Tomáš Votruba
Nikita Popov
parent
b9b45dd2bc
commit
3f718ee2c3
@ -7,6 +7,7 @@ use PhpParser\ErrorHandler;
|
||||
use PhpParser\Lexer;
|
||||
use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
|
||||
use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
|
||||
use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
|
||||
use PhpParser\Lexer\TokenEmulator\TokenEmulatorInterface;
|
||||
use PhpParser\Parser\Tokens;
|
||||
|
||||
@ -39,6 +40,7 @@ REGEX;
|
||||
|
||||
$this->tokenEmulators[] = new FnTokenEmulator();
|
||||
$this->tokenEmulators[] = new CoaleseEqualTokenEmulator();
|
||||
$this->tokenEmulators[] = new NumericLiteralSeparatorEmulator();
|
||||
|
||||
$this->tokenMap[self::T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
|
||||
$this->tokenMap[self::T_FN] = Tokens::T_FN;
|
||||
@ -58,14 +60,6 @@ REGEX;
|
||||
// 1. emulation of heredoc and nowdoc new syntax
|
||||
$preparedCode = $this->processHeredocNowdoc($code);
|
||||
parent::startLexing($preparedCode, $collector);
|
||||
|
||||
// add token emulation
|
||||
foreach ($this->tokenEmulators as $emulativeToken) {
|
||||
if ($emulativeToken->isEmulationNeeded($code)) {
|
||||
$this->tokens = $emulativeToken->emulate($code, $this->tokens);
|
||||
}
|
||||
}
|
||||
|
||||
$this->fixupTokens();
|
||||
|
||||
$errors = $collector->getErrors();
|
||||
@ -75,6 +69,13 @@ REGEX;
|
||||
$errorHandler->handleError($error);
|
||||
}
|
||||
}
|
||||
|
||||
// add token emulation
|
||||
foreach ($this->tokenEmulators as $emulativeToken) {
|
||||
if ($emulativeToken->isEmulationNeeded($code)) {
|
||||
$this->tokens = $emulativeToken->emulate($code, $this->tokens);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function isHeredocNowdocEmulationNeeded(string $code): bool
|
||||
|
||||
@ -0,0 +1,98 @@
|
||||
<?php declare(strict_types=1);
|
||||
|
||||
namespace PhpParser\Lexer\TokenEmulator;
|
||||
|
||||
use PhpParser\Lexer\Emulative;
|
||||
|
||||
final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
|
||||
{
|
||||
const BIN = '(?:0b[01]+(?:_[01]+)*)';
|
||||
const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
|
||||
const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
|
||||
const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
|
||||
const EXP = '(?:e[+-]?' . self::DEC . ')';
|
||||
const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
|
||||
const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
|
||||
|
||||
public function isEmulationNeeded(string $code) : bool
|
||||
{
|
||||
// skip version where this is supported
|
||||
if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return preg_match('~[0-9a-f]_[0-9a-f]~i', $code) !== false;
|
||||
}
|
||||
|
||||
public function emulate(string $code, array $tokens): array
|
||||
{
|
||||
// We need to manually iterate and manage a count because we'll change
|
||||
// the tokens array on the way
|
||||
$codeOffset = 0;
|
||||
for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
|
||||
$token = $tokens[$i];
|
||||
$tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
|
||||
|
||||
if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
|
||||
$codeOffset += $tokenLen;
|
||||
continue;
|
||||
}
|
||||
|
||||
$res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
|
||||
assert($res, "No number at number token position");
|
||||
|
||||
$match = $matches[0];
|
||||
$matchLen = \strlen($match);
|
||||
if ($matchLen === $tokenLen) {
|
||||
// Original token already holds the full number.
|
||||
$codeOffset += $tokenLen;
|
||||
continue;
|
||||
}
|
||||
|
||||
$tokenKind = $this->resolveIntegerOrFloatToken($match);
|
||||
$newTokens = [[$tokenKind, $match, $token[2]]];
|
||||
|
||||
$numTokens = 1;
|
||||
$len = $tokenLen;
|
||||
while ($matchLen > $len) {
|
||||
$nextToken = $tokens[$i + $numTokens];
|
||||
$nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
|
||||
$nextTokenLen = \strlen($nextTokenText);
|
||||
|
||||
$numTokens++;
|
||||
if ($matchLen < $len + $nextTokenLen) {
|
||||
// Split trailing characters into a partial token.
|
||||
assert(is_array($nextToken), "Partial token should be an array token");
|
||||
$partialText = substr($nextTokenText, $matchLen - $len);
|
||||
$newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
|
||||
break;
|
||||
}
|
||||
|
||||
$len += $nextTokenLen;
|
||||
}
|
||||
|
||||
array_splice($tokens, $i, $numTokens, $newTokens);
|
||||
$c -= $numTokens - \count($newTokens);
|
||||
$codeOffset += $matchLen;
|
||||
}
|
||||
|
||||
return $tokens;
|
||||
}
|
||||
|
||||
private function resolveIntegerOrFloatToken(string $str): int
|
||||
{
|
||||
$str = str_replace('_', '', $str);
|
||||
|
||||
if (stripos($str, '0b') === 0) {
|
||||
$num = bindec($str);
|
||||
} elseif (stripos($str, '0x') === 0) {
|
||||
$num = hexdec($str);
|
||||
} elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
|
||||
$num = octdec($str);
|
||||
} else {
|
||||
$num = +$str;
|
||||
}
|
||||
|
||||
return is_float($num) ? T_DNUMBER : T_LNUMBER;
|
||||
}
|
||||
}
|
||||
@ -34,6 +34,8 @@ class DNumber extends Scalar
|
||||
* @return float The parsed number
|
||||
*/
|
||||
public static function parse(string $str) : float {
|
||||
$str = str_replace('_', '', $str);
|
||||
|
||||
// if string contains any of .eE just cast it to float
|
||||
if (false !== strpbrk($str, '.eE')) {
|
||||
return (float) $str;
|
||||
|
||||
@ -41,6 +41,8 @@ class LNumber extends Scalar
|
||||
* @return LNumber The constructed LNumber, including kind attribute
|
||||
*/
|
||||
public static function fromString(string $str, array $attributes = [], bool $allowInvalidOctal = false) : LNumber {
|
||||
$str = str_replace('_', '', $str);
|
||||
|
||||
if ('0' !== $str[0] || '0' === $str) {
|
||||
$attributes['kind'] = LNumber::KIND_DEC;
|
||||
return new LNumber((int) $str, $attributes);
|
||||
|
||||
@ -123,10 +123,6 @@ class EmulativeTest extends LexerTest
|
||||
|
||||
public function provideTestLexNewFeatures() {
|
||||
return [
|
||||
// PHP 7.4
|
||||
['??=', [
|
||||
[Tokens::T_COALESCE_EQUAL, '??='],
|
||||
]],
|
||||
['yield from', [
|
||||
[Tokens::T_YIELD_FROM, 'yield from'],
|
||||
]],
|
||||
@ -169,7 +165,7 @@ class EmulativeTest extends LexerTest
|
||||
[ord(';'), ';'],
|
||||
]],
|
||||
|
||||
// Flexible heredoc/nowdoc
|
||||
// PHP 7.3: Flexible heredoc/nowdoc
|
||||
["<<<LABEL\nLABEL,", [
|
||||
[Tokens::T_START_HEREDOC, "<<<LABEL\n"],
|
||||
[Tokens::T_END_HEREDOC, "LABEL"],
|
||||
@ -205,6 +201,58 @@ class EmulativeTest extends LexerTest
|
||||
[Tokens::T_END_HEREDOC, " LABEL"],
|
||||
[Tokens::T_STRING, "LABEL"],
|
||||
]],
|
||||
|
||||
// PHP 7.4: Null coalesce equal
|
||||
['??=', [
|
||||
[Tokens::T_COALESCE_EQUAL, '??='],
|
||||
]],
|
||||
|
||||
// PHP 7.4: Number literal separator
|
||||
['1_000', [
|
||||
[Tokens::T_LNUMBER, '1_000'],
|
||||
]],
|
||||
['0xCAFE_F00D', [
|
||||
[Tokens::T_LNUMBER, '0xCAFE_F00D'],
|
||||
]],
|
||||
['0b0101_1111', [
|
||||
[Tokens::T_LNUMBER, '0b0101_1111'],
|
||||
]],
|
||||
['0137_041', [
|
||||
[Tokens::T_LNUMBER, '0137_041'],
|
||||
]],
|
||||
['1_000.0', [
|
||||
[Tokens::T_DNUMBER, '1_000.0'],
|
||||
]],
|
||||
['1_0.0', [
|
||||
[Tokens::T_DNUMBER, '1_0.0']
|
||||
]],
|
||||
['1_000_000_000.0', [
|
||||
[Tokens::T_DNUMBER, '1_000_000_000.0']
|
||||
]],
|
||||
['0e1_0', [
|
||||
[Tokens::T_DNUMBER, '0e1_0']
|
||||
]],
|
||||
['1_0e+10', [
|
||||
[Tokens::T_DNUMBER, '1_0e+10']
|
||||
]],
|
||||
['1_0e-10', [
|
||||
[Tokens::T_DNUMBER, '1_0e-10']
|
||||
]],
|
||||
['0b1011010101001010_110101010010_10101101010101_0101101011001_110111100', [
|
||||
[Tokens::T_DNUMBER, '0b1011010101001010_110101010010_10101101010101_0101101011001_110111100'],
|
||||
]],
|
||||
['0xFFFF_FFFF_FFFF_FFFF', [
|
||||
[Tokens::T_DNUMBER, '0xFFFF_FFFF_FFFF_FFFF'],
|
||||
]],
|
||||
['1_000+1', [
|
||||
[Tokens::T_LNUMBER, '1_000'],
|
||||
[ord('+'), '+'],
|
||||
[Tokens::T_LNUMBER, '1'],
|
||||
]],
|
||||
['1_0abc', [
|
||||
[Tokens::T_LNUMBER, '1_0'],
|
||||
[Tokens::T_STRING, 'abc'],
|
||||
]],
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
199
test/code/parser/scalar/numberSeparators.test
Normal file
199
test/code/parser/scalar/numberSeparators.test
Normal file
@ -0,0 +1,199 @@
|
||||
Different integer syntaxes
|
||||
-----
|
||||
<?php
|
||||
|
||||
6.674_083e-11;
|
||||
299_792_458;
|
||||
0xCAFE_F00D;
|
||||
0b0101_1111;
|
||||
0137_041;
|
||||
|
||||
// already a valid constant name
|
||||
_100;
|
||||
|
||||
// syntax errors
|
||||
100_;
|
||||
1__1;
|
||||
1_.0;
|
||||
1._0;
|
||||
0x_123;
|
||||
0b_101;
|
||||
1_e2;
|
||||
1e_2;
|
||||
-----
|
||||
Syntax error, unexpected T_STRING from 13:4 to 13:4
|
||||
Syntax error, unexpected T_STRING from 14:2 to 14:4
|
||||
Syntax error, unexpected T_STRING from 15:2 to 15:2
|
||||
Syntax error, unexpected T_STRING from 16:3 to 16:4
|
||||
Syntax error, unexpected T_STRING from 17:2 to 17:6
|
||||
Syntax error, unexpected T_STRING from 18:2 to 18:6
|
||||
Syntax error, unexpected T_STRING from 19:2 to 19:4
|
||||
Syntax error, unexpected T_STRING from 20:2 to 20:4
|
||||
array(
|
||||
0: Stmt_Expression(
|
||||
expr: Scalar_DNumber(
|
||||
value: 6.674083E-11
|
||||
)
|
||||
)
|
||||
1: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 299792458
|
||||
)
|
||||
)
|
||||
2: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 3405705229
|
||||
)
|
||||
)
|
||||
3: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 95
|
||||
)
|
||||
)
|
||||
4: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 48673
|
||||
)
|
||||
)
|
||||
5: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: _100
|
||||
)
|
||||
comments: array(
|
||||
0: // already a valid constant name
|
||||
)
|
||||
)
|
||||
comments: array(
|
||||
0: // already a valid constant name
|
||||
)
|
||||
)
|
||||
comments: array(
|
||||
0: // already a valid constant name
|
||||
)
|
||||
)
|
||||
6: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 100
|
||||
comments: array(
|
||||
0: // syntax errors
|
||||
)
|
||||
)
|
||||
comments: array(
|
||||
0: // syntax errors
|
||||
)
|
||||
)
|
||||
7: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: _
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
8: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 1
|
||||
)
|
||||
)
|
||||
9: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: __1
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
10: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 1
|
||||
)
|
||||
)
|
||||
11: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: _
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
12: Stmt_Expression(
|
||||
expr: Scalar_DNumber(
|
||||
value: 0
|
||||
)
|
||||
)
|
||||
13: Stmt_Expression(
|
||||
expr: Scalar_DNumber(
|
||||
value: 1
|
||||
)
|
||||
)
|
||||
14: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: _0
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
15: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 0
|
||||
)
|
||||
)
|
||||
16: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: x_123
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
17: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 0
|
||||
)
|
||||
)
|
||||
18: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: b_101
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
19: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 1
|
||||
)
|
||||
)
|
||||
20: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: _e2
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
21: Stmt_Expression(
|
||||
expr: Scalar_LNumber(
|
||||
value: 1
|
||||
)
|
||||
)
|
||||
22: Stmt_Expression(
|
||||
expr: Expr_ConstFetch(
|
||||
name: Name(
|
||||
parts: array(
|
||||
0: e_2
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
@ -231,6 +231,8 @@ foreach (new RecursiveIteratorIterator(
|
||||
echo $file, ":\n Parse failed with message: {$e->getMessage()}\n";
|
||||
|
||||
++$parseFail;
|
||||
} catch (Throwable $e) {
|
||||
echo $file, ":\n Unknown error occurred: $e\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user