[PHP 7.4] Add support for numeric literal separators (#615)

Implements RFC https://wiki.php.net/rfc/numeric_literal_separator.

Closes #614.

lib/PhpParser/Lexer/Emulative.php

@@ -7,6 +7,7 @@ use PhpParser\ErrorHandler;
 use PhpParser\Lexer;
 use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
 use PhpParser\Lexer\TokenEmulator\TokenEmulatorInterface;
 use PhpParser\Parser\Tokens;
 
@@ -39,6 +40,7 @@ REGEX;
 
         $this->tokenEmulators[] = new FnTokenEmulator();
         $this->tokenEmulators[] = new CoaleseEqualTokenEmulator();
+        $this->tokenEmulators[] = new NumericLiteralSeparatorEmulator();
 
         $this->tokenMap[self::T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
         $this->tokenMap[self::T_FN] = Tokens::T_FN;
@@ -58,14 +60,6 @@ REGEX;
         // 1. emulation of heredoc and nowdoc new syntax
         $preparedCode = $this->processHeredocNowdoc($code);
         parent::startLexing($preparedCode, $collector);
-
-        // add token emulation
-        foreach ($this->tokenEmulators as $emulativeToken) {
-            if ($emulativeToken->isEmulationNeeded($code)) {
-                $this->tokens = $emulativeToken->emulate($code, $this->tokens);
-            }
-        }
-
         $this->fixupTokens();
 
         $errors = $collector->getErrors();
@@ -75,6 +69,13 @@ REGEX;
                 $errorHandler->handleError($error);
             }
         }
+
+        // add token emulation
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                $this->tokens = $emulativeToken->emulate($code, $this->tokens);
+            }
+        }
     }
 
     private function isHeredocNowdocEmulationNeeded(string $code): bool

lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -0,0 +1,98 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
+{
+    const BIN = '(?:0b[01]+(?:_[01]+)*)';
+    const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
+    const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
+    const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
+    const EXP = '(?:e[+-]?' . self::DEC . ')';
+    const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
+    const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return preg_match('~[0-9a-f]_[0-9a-f]~i', $code) !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $codeOffset = 0;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            $tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
+
+            if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
+            assert($res, "No number at number token position");
+
+            $match = $matches[0];
+            $matchLen = \strlen($match);
+            if ($matchLen === $tokenLen) {
+                // Original token already holds the full number.
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $tokenKind = $this->resolveIntegerOrFloatToken($match);
+            $newTokens = [[$tokenKind, $match, $token[2]]];
+
+            $numTokens = 1;
+            $len = $tokenLen;
+            while ($matchLen > $len) {
+                $nextToken = $tokens[$i + $numTokens];
+                $nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
+                $nextTokenLen = \strlen($nextTokenText);
+
+                $numTokens++;
+                if ($matchLen < $len + $nextTokenLen) {
+                    // Split trailing characters into a partial token.
+                    assert(is_array($nextToken), "Partial token should be an array token");
+                    $partialText = substr($nextTokenText, $matchLen - $len);
+                    $newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
+                    break;
+                }
+
+                $len += $nextTokenLen;
+            }
+
+            array_splice($tokens, $i, $numTokens, $newTokens);
+            $c -= $numTokens - \count($newTokens);
+            $codeOffset += $matchLen;
+        }
+
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int
+    {
+        $str = str_replace('_', '', $str);
+
+        if (stripos($str, '0b') === 0) {
+            $num = bindec($str);
+        } elseif (stripos($str, '0x') === 0) {
+            $num = hexdec($str);
+        } elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
+            $num = octdec($str);
+        } else {
+            $num = +$str;
+        }
+
+        return is_float($num) ? T_DNUMBER : T_LNUMBER;
+    }
+}

lib/PhpParser/Node/Scalar/DNumber.php

@@ -34,6 +34,8 @@ class DNumber extends Scalar
      * @return float The parsed number
      */
     public static function parse(string $str) : float {
+        $str = str_replace('_', '', $str);
+
         // if string contains any of .eE just cast it to float
         if (false !== strpbrk($str, '.eE')) {
             return (float) $str;

lib/PhpParser/Node/Scalar/LNumber.php

@@ -41,6 +41,8 @@ class LNumber extends Scalar
      * @return LNumber The constructed LNumber, including kind attribute
      */
     public static function fromString(string $str, array $attributes = [], bool $allowInvalidOctal = false) : LNumber {
+        $str = str_replace('_', '', $str);
+
         if ('0' !== $str[0] || '0' === $str) {
             $attributes['kind'] = LNumber::KIND_DEC;
             return new LNumber((int) $str, $attributes);

test/PhpParser/Lexer/EmulativeTest.php

@@ -123,10 +123,6 @@ class EmulativeTest extends LexerTest
 
     public function provideTestLexNewFeatures() {
         return [
-            // PHP 7.4
-            ['??=', [
-                [Tokens::T_COALESCE_EQUAL, '??='],
-            ]],
             ['yield from', [
                 [Tokens::T_YIELD_FROM, 'yield from'],
             ]],
@@ -169,7 +165,7 @@ class EmulativeTest extends LexerTest
                 [ord(';'), ';'],
             ]],
 
-            // Flexible heredoc/nowdoc
+            // PHP 7.3: Flexible heredoc/nowdoc
             ["<<<LABEL\nLABEL,", [
                 [Tokens::T_START_HEREDOC, "<<<LABEL\n"],
                 [Tokens::T_END_HEREDOC, "LABEL"],
@@ -205,6 +201,58 @@ class EmulativeTest extends LexerTest
                 [Tokens::T_END_HEREDOC, "    LABEL"],
                 [Tokens::T_STRING, "LABEL"],
             ]],
+
+            // PHP 7.4: Null coalesce equal
+            ['??=', [
+                [Tokens::T_COALESCE_EQUAL, '??='],
+            ]],
+
+            // PHP 7.4: Number literal separator
+            ['1_000', [
+                [Tokens::T_LNUMBER, '1_000'],
+            ]],
+            ['0xCAFE_F00D', [
+                [Tokens::T_LNUMBER, '0xCAFE_F00D'],
+            ]],
+            ['0b0101_1111', [
+                [Tokens::T_LNUMBER, '0b0101_1111'],
+            ]],
+            ['0137_041', [
+                [Tokens::T_LNUMBER, '0137_041'],
+            ]],
+            ['1_000.0', [
+                [Tokens::T_DNUMBER, '1_000.0'],
+            ]],
+            ['1_0.0', [
+                [Tokens::T_DNUMBER, '1_0.0']
+            ]],
+            ['1_000_000_000.0', [
+                [Tokens::T_DNUMBER, '1_000_000_000.0']
+            ]],
+            ['0e1_0', [
+                [Tokens::T_DNUMBER, '0e1_0']
+            ]],
+            ['1_0e+10', [
+                [Tokens::T_DNUMBER, '1_0e+10']
+            ]],
+            ['1_0e-10', [
+                [Tokens::T_DNUMBER, '1_0e-10']
+            ]],
+            ['0b1011010101001010_110101010010_10101101010101_0101101011001_110111100', [
+                [Tokens::T_DNUMBER, '0b1011010101001010_110101010010_10101101010101_0101101011001_110111100'],
+            ]],
+            ['0xFFFF_FFFF_FFFF_FFFF', [
+                [Tokens::T_DNUMBER, '0xFFFF_FFFF_FFFF_FFFF'],
+            ]],
+            ['1_000+1', [
+                [Tokens::T_LNUMBER, '1_000'],
+                [ord('+'), '+'],
+                [Tokens::T_LNUMBER, '1'],
+            ]],
+            ['1_0abc', [
+                [Tokens::T_LNUMBER, '1_0'],
+                [Tokens::T_STRING, 'abc'],
+            ]],
         ];
     }
 }

test/code/parser/scalar/numberSeparators.test

@@ -0,0 +1,199 @@
+Different integer syntaxes
+-----
+<?php
+
+6.674_083e-11;
+299_792_458;
+0xCAFE_F00D;
+0b0101_1111;
+0137_041;
+
+// already a valid constant name
+_100;
+
+// syntax errors
+100_;
+1__1;
+1_.0;
+1._0;
+0x_123;
+0b_101;
+1_e2;
+1e_2;
+-----
+Syntax error, unexpected T_STRING from 13:4 to 13:4
+Syntax error, unexpected T_STRING from 14:2 to 14:4
+Syntax error, unexpected T_STRING from 15:2 to 15:2
+Syntax error, unexpected T_STRING from 16:3 to 16:4
+Syntax error, unexpected T_STRING from 17:2 to 17:6
+Syntax error, unexpected T_STRING from 18:2 to 18:6
+Syntax error, unexpected T_STRING from 19:2 to 19:4
+Syntax error, unexpected T_STRING from 20:2 to 20:4
+array(
+    0: Stmt_Expression(
+        expr: Scalar_DNumber(
+            value: 6.674083E-11
+        )
+    )
+    1: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 299792458
+        )
+    )
+    2: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 3405705229
+        )
+    )
+    3: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 95
+        )
+    )
+    4: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 48673
+        )
+    )
+    5: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: _100
+                )
+                comments: array(
+                    0: // already a valid constant name
+                )
+            )
+            comments: array(
+                0: // already a valid constant name
+            )
+        )
+        comments: array(
+            0: // already a valid constant name
+        )
+    )
+    6: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 100
+            comments: array(
+                0: // syntax errors
+            )
+        )
+        comments: array(
+            0: // syntax errors
+        )
+    )
+    7: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: _
+                )
+            )
+        )
+    )
+    8: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 1
+        )
+    )
+    9: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: __1
+                )
+            )
+        )
+    )
+    10: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 1
+        )
+    )
+    11: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: _
+                )
+            )
+        )
+    )
+    12: Stmt_Expression(
+        expr: Scalar_DNumber(
+            value: 0
+        )
+    )
+    13: Stmt_Expression(
+        expr: Scalar_DNumber(
+            value: 1
+        )
+    )
+    14: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: _0
+                )
+            )
+        )
+    )
+    15: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 0
+        )
+    )
+    16: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: x_123
+                )
+            )
+        )
+    )
+    17: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 0
+        )
+    )
+    18: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: b_101
+                )
+            )
+        )
+    )
+    19: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 1
+        )
+    )
+    20: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: _e2
+                )
+            )
+        )
+    )
+    21: Stmt_Expression(
+        expr: Scalar_LNumber(
+            value: 1
+        )
+    )
+    22: Stmt_Expression(
+        expr: Expr_ConstFetch(
+            name: Name(
+                parts: array(
+                    0: e_2
+                )
+            )
+        )
+    )
+)

test_old/run.php

@@ -231,6 +231,8 @@ foreach (new RecursiveIteratorIterator(
         echo $file, ":\n    Parse failed with message: {$e->getMessage()}\n";
 
         ++$parseFail;
+    } catch (Throwable $e) {
+        echo $file, ":\n    Unknown error occurred: $e\n";
     }
 }