Release PHP-Parser 4.15.4

Test taken from PR #797.

(cherry picked from commit 9b46dffb12)

.github/workflows/main.yml

@@ -40,9 +40,7 @@ jobs:
           - "7.4"
           - "8.0"
           - "8.1"
-        include:
-          - php-version: "8.1"
-            flags: "--ignore-platform-req=php"
+          - "8.2"
     steps:
       - name: "Checkout"
         uses: "actions/checkout@v2"

CHANGELOG.md

@@ -1,3 +1,28 @@
+Version 4.15.4 (2023-03-05)
+---------------------------
+
+### Fixed
+
+* Fixed formatting-preservation for alternative if syntax with trailing comments.
+
+Version 4.15.3 (2023-01-16)
+---------------------------
+
+### Fixed
+
+* Support readonly property with PHP 8.2 DNF type.
+* Fixed PHP attribute group and PHP-Parser attribute mixup in EnumCase builder.
+
+Version 4.15.2 (2022-11-12)
+---------------------------
+
+### Fixed
+
+* Fixed parsing of large hex float literals that contain an "e" character.
+* Fixed tests to pass on 32-bit.
+* Fixed generation of invalid code when using formatting-preserving pretty printer with code that
+  uses inline HTML.
+
 Version 4.15.1 (2022-09-04)
 ---------------------------
 

grammar/php5.y

@@ -264,8 +264,13 @@ optional_ellipsis:
     | T_ELLIPSIS                                            { $$ = true; }
 ;
 
+identifier_maybe_readonly:
+      identifier                                            { $$ = $1; }
+    | T_READONLY                                            { $$ = Node\Identifier[$1]; }
+;
+
 function_declaration_statement:
-    T_FUNCTION optional_ref identifier '(' parameter_list ')' optional_return_type '{' inner_statement_list '}'
+    T_FUNCTION optional_ref identifier_maybe_readonly '(' parameter_list ')' optional_return_type '{' inner_statement_list '}'
         { $$ = Stmt\Function_[$3, ['byRef' => $2, 'params' => $5, 'returnType' => $7, 'stmts' => $9]]; }
 ;
 
@@ -721,8 +726,13 @@ lexical_var:
       optional_ref plain_variable                           { $$ = Expr\ClosureUse[$2, $1]; }
 ;
 
+name_readonly:
+      T_READONLY                                            { $$ = Name[$1]; }
+;
+
 function_call:
       name argument_list                                    { $$ = Expr\FuncCall[$1, $2]; }
+    | name_readonly argument_list                           { $$ = Expr\FuncCall[$1, $2]; }
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM identifier_ex argument_list
           { $$ = Expr\StaticCall[$1, $3, $4]; }
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM '{' expr '}' argument_list

grammar/php7.y

@@ -350,15 +350,23 @@ block_or_error:
     | error                                                 { $$ = []; }
 ;
 
+identifier_maybe_readonly:
+      identifier_not_reserved                               { $$ = $1; }
+    | T_READONLY                                            { $$ = Node\Identifier[$1]; }
+;
+
 function_declaration_statement:
-      T_FUNCTION optional_ref identifier_not_reserved '(' parameter_list ')' optional_return_type block_or_error
+      T_FUNCTION optional_ref identifier_maybe_readonly '(' parameter_list ')' optional_return_type block_or_error
           { $$ = Stmt\Function_[$3, ['byRef' => $2, 'params' => $5, 'returnType' => $7, 'stmts' => $8, 'attrGroups' => []]]; }
-    | attributes T_FUNCTION optional_ref identifier_not_reserved '(' parameter_list ')' optional_return_type block_or_error
+    | attributes T_FUNCTION optional_ref identifier_maybe_readonly '(' parameter_list ')' optional_return_type block_or_error
           { $$ = Stmt\Function_[$4, ['byRef' => $3, 'params' => $6, 'returnType' => $8, 'stmts' => $9, 'attrGroups' => $1]]; }
 ;
 
 class_declaration_statement:
-      optional_attributes class_entry_type identifier_not_reserved extends_from implements_list '{' class_statement_list '}'
+      class_entry_type identifier_not_reserved extends_from implements_list '{' class_statement_list '}'
+          { $$ = Stmt\Class_[$2, ['type' => $1, 'extends' => $3, 'implements' => $4, 'stmts' => $6, 'attrGroups' => []]];
+            $this->checkClass($$, #2); }
+    | attributes class_entry_type identifier_not_reserved extends_from implements_list '{' class_statement_list '}'
           { $$ = Stmt\Class_[$3, ['type' => $2, 'extends' => $4, 'implements' => $5, 'stmts' => $7, 'attrGroups' => $1]];
             $this->checkClass($$, #3); }
     | optional_attributes T_INTERFACE identifier_not_reserved interface_extends_list '{' class_statement_list '}'
@@ -510,7 +518,8 @@ new_elseif_list:
 ;
 
 new_elseif:
-     T_ELSEIF '(' expr ')' ':' inner_statement_list         { $$ = Stmt\ElseIf_[$3, $6]; }
+     T_ELSEIF '(' expr ')' ':' inner_statement_list
+         { $$ = Stmt\ElseIf_[$3, $6]; $this->fixupAlternativeElse($$); }
 ;
 
 else_single:
@@ -520,7 +529,8 @@ else_single:
 
 new_else_single:
       /* empty */                                           { $$ = null; }
-    | T_ELSE ':' inner_statement_list                       { $$ = Stmt\Else_[$3]; }
+    | T_ELSE ':' inner_statement_list
+          { $$ = Stmt\Else_[$3]; $this->fixupAlternativeElse($$); }
 ;
 
 foreach_variable:
@@ -962,8 +972,13 @@ lexical_var:
       optional_ref plain_variable                           { $$ = Expr\ClosureUse[$2, $1]; }
 ;
 
+name_readonly:
+      T_READONLY                                            { $$ = Name[$1]; }
+;
+
 function_call:
       name argument_list                                    { $$ = Expr\FuncCall[$1, $2]; }
+    | name_readonly argument_list                           { $$ = Expr\FuncCall[$1, $2]; }
     | callable_expr argument_list                           { $$ = Expr\FuncCall[$1, $2]; }
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM member_name argument_list
           { $$ = Expr\StaticCall[$1, $3, $4]; }

lib/PhpParser/Builder/EnumCase.php

@@ -78,8 +78,8 @@ class EnumCase implements PhpParser\Builder
         return new Stmt\EnumCase(
             $this->name,
             $this->value,
-            $this->attributes,
-            $this->attributeGroups
+            $this->attributeGroups,
+            $this->attributes
         );
     }
 }

lib/PhpParser/Internal/TokenStream.php

@@ -206,6 +206,11 @@ class TokenStream
             || $this->haveTokenInRange($startPos, $endPos, '}');
     }
 
+    public function haveTagInRange(int $startPos, int $endPos): bool {
+        return $this->haveTokenInRange($startPos, $endPos, \T_OPEN_TAG)
+            || $this->haveTokenInRange($startPos, $endPos, \T_CLOSE_TAG);
+    }
+
     /**
      * Get indentation before token position.
      *

lib/PhpParser/Lexer/Emulative.php

@@ -14,6 +14,7 @@ use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\MatchTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\NullsafeTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
+use PhpParser\Lexer\TokenEmulator\ReadonlyFunctionTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\ReadonlyTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\ReverseEmulator;
 use PhpParser\Lexer\TokenEmulator\TokenEmulator;
@@ -24,6 +25,7 @@ class Emulative extends Lexer
     const PHP_7_4 = '7.4dev';
     const PHP_8_0 = '8.0dev';
     const PHP_8_1 = '8.1dev';
+    const PHP_8_2 = '8.2dev';
 
     /** @var mixed[] Patches used to reverse changes introduced in the code */
     private $patches = [];
@@ -41,7 +43,7 @@ class Emulative extends Lexer
      */
     public function __construct(array $options = [])
     {
-        $this->targetPhpVersion = $options['phpVersion'] ?? Emulative::PHP_8_1;
+        $this->targetPhpVersion = $options['phpVersion'] ?? Emulative::PHP_8_2;
         unset($options['phpVersion']);
 
         parent::__construct($options);
@@ -57,6 +59,7 @@ class Emulative extends Lexer
             new EnumTokenEmulator(),
             new ReadonlyTokenEmulator(),
             new ExplicitOctalEmulator(),
+            new ReadonlyFunctionTokenEmulator(),
         ];
 
         // Collect emulators that are relevant for the PHP version we're running

lib/PhpParser/Lexer/TokenEmulator/ReadonlyFunctionTokenEmulator.php

@@ -0,0 +1,31 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+/*
+ * In PHP 8.1, "readonly(" was special cased in the lexer in order to support functions with
+ * name readonly. In PHP 8.2, this may conflict with readonly properties having a DNF type. For
+ * this reason, PHP 8.2 instead treats this as T_READONLY and then handles it specially in the
+ * parser. This emulator only exists to handle this special case, which is skipped by the
+ * PHP 8.1 ReadonlyTokenEmulator.
+ */
+class ReadonlyFunctionTokenEmulator extends KeywordEmulator {
+    public function getKeywordString(): string {
+        return 'readonly';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_READONLY;
+    }
+
+    public function getPhpVersion(): string {
+        return Emulative::PHP_8_2;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // Don't bother
+        return $tokens;
+    }
+}

lib/PhpParser/Node/Scalar/DNumber.php

@@ -47,13 +47,7 @@ class DNumber extends Scalar
     public static function parse(string $str) : float {
         $str = str_replace('_', '', $str);
 
-        // if string contains any of .eE just cast it to float
-        if (false !== strpbrk($str, '.eE')) {
-            return (float) $str;
-        }
-
-        // otherwise it's an integer notation that overflowed into a float
-        // if it starts with 0 it's one of the special integer notations
+        // Check whether this is one of the special integer notations.
         if ('0' === $str[0]) {
             // hex
             if ('x' === $str[1] || 'X' === $str[1]) {
@@ -65,10 +59,12 @@ class DNumber extends Scalar
                 return bindec($str);
             }
 
-            // oct
-            // substr($str, 0, strcspn($str, '89')) cuts the string at the first invalid digit (8 or 9)
-            // so that only the digits before that are used
-            return octdec(substr($str, 0, strcspn($str, '89')));
+            // oct, but only if the string does not contain any of '.eE'.
+            if (false === strpbrk($str, '.eE')) {
+                // substr($str, 0, strcspn($str, '89')) cuts the string at the first invalid digit
+                // (8 or 9) so that only the digits before that are used.
+                return octdec(substr($str, 0, strcspn($str, '89')));
+            }
         }
 
         // dec

lib/PhpParser/Node/UnionType.php

@@ -4,7 +4,7 @@ namespace PhpParser\Node;
 
 class UnionType extends ComplexType
 {
-    /** @var (Identifier|Name)[] Types */
+    /** @var (Identifier|Name|IntersectionType)[] Types */
     public $types;
 
     /**

lib/PhpParser/NodeVisitor/NameResolver.php

@@ -161,7 +161,7 @@ class NameResolver extends NodeVisitorAbstract
         return null;
     }
 
-    private function addAlias(Stmt\UseUse $use, $type, Name $prefix = null) {
+    private function addAlias(Stmt\UseUse $use, int $type, Name $prefix = null) {
         // Add prefix for group uses
         $name = $prefix ? Name::concat($prefix, $use->name) : $use->name;
         // Type is determined either by individual element or whole use declaration

lib/PhpParser/ParserAbstract.php

@@ -16,9 +16,12 @@ use PhpParser\Node\Scalar\String_;
 use PhpParser\Node\Stmt\Class_;
 use PhpParser\Node\Stmt\ClassConst;
 use PhpParser\Node\Stmt\ClassMethod;
+use PhpParser\Node\Stmt\Else_;
+use PhpParser\Node\Stmt\ElseIf_;
 use PhpParser\Node\Stmt\Enum_;
 use PhpParser\Node\Stmt\Interface_;
 use PhpParser\Node\Stmt\Namespace_;
+use PhpParser\Node\Stmt\Nop;
 use PhpParser\Node\Stmt\Property;
 use PhpParser\Node\Stmt\TryCatch;
 use PhpParser\Node\Stmt\UseUse;
@@ -876,6 +879,24 @@ abstract class ParserAbstract implements Parser
         return $attributes;
     }
 
+    /** @param ElseIf_|Else_ $node */
+    protected function fixupAlternativeElse($node) {
+        // Make sure a trailing nop statement carrying comments is part of the node.
+        $numStmts = \count($node->stmts);
+        if ($numStmts !== 0 && $node->stmts[$numStmts - 1] instanceof Nop) {
+            $nopAttrs = $node->stmts[$numStmts - 1]->getAttributes();
+            if (isset($nopAttrs['endLine'])) {
+                $node->setAttribute('endLine', $nopAttrs['endLine']);
+            }
+            if (isset($nopAttrs['endFilePos'])) {
+                $node->setAttribute('endFilePos', $nopAttrs['endFilePos']);
+            }
+            if (isset($nopAttrs['endTokenPos'])) {
+                $node->setAttribute('endTokenPos', $nopAttrs['endTokenPos']);
+            }
+        }
+    }
+
     protected function checkClassModifier($a, $b, $modifierPos) {
         try {
             Class_::verifyClassModifier($a, $b);

lib/PhpParser/PrettyPrinterAbstract.php

@@ -774,7 +774,8 @@ abstract class PrettyPrinterAbstract
                 }
 
                 if ($skipRemovedNode) {
-                    if ($isStmtList && $this->origTokens->haveBracesInRange($pos, $itemStartPos)) {
+                    if ($isStmtList && ($this->origTokens->haveBracesInRange($pos, $itemStartPos) ||
+                                        $this->origTokens->haveTagInRange($pos, $itemStartPos))) {
                         // We'd remove the brace of a code block.
                         // TODO: Preserve formatting.
                         $this->setIndentLevel($origIndentLevel);
@@ -877,7 +878,8 @@ abstract class PrettyPrinterAbstract
                         $pos, $itemStartPos, $indentAdjustment);
                     $skipRemovedNode = true;
                 } else {
-                    if ($isStmtList && $this->origTokens->haveBracesInRange($pos, $itemStartPos)) {
+                    if ($isStmtList && ($this->origTokens->haveBracesInRange($pos, $itemStartPos) ||
+                                        $this->origTokens->haveTagInRange($pos, $itemStartPos))) {
                         // We'd remove the brace of a code block.
                         // TODO: Preserve formatting.
                         return null;

test/PhpParser/Builder/EnumCaseTest.php

@@ -27,6 +27,7 @@ class EnumCaseTest extends \PHPUnit\Framework\TestCase
             new Stmt\EnumCase(
                 "TEST",
                 null,
+                [],
                 [
                     'comments' => [new Comment\Doc('/** Test */')]
                 ]
@@ -50,7 +51,6 @@ class EnumCaseTest extends \PHPUnit\Framework\TestCase
             new Stmt\EnumCase(
                 "ATTR_GROUP",
                 null,
-                [],
                 [$attributeGroup]
             ),
             $node

test/PhpParser/Lexer/EmulativeTest.php

@@ -241,8 +241,8 @@ class EmulativeTest extends LexerTest
             ['1_000', [
                 [Tokens::T_LNUMBER, '1_000'],
             ]],
-            ['0xCAFE_F00D', [
-                [Tokens::T_LNUMBER, '0xCAFE_F00D'],
+            ['0x7AFE_F00D', [
+                [Tokens::T_LNUMBER, '0x7AFE_F00D'],
             ]],
             ['0b0101_1111', [
                 [Tokens::T_LNUMBER, '0b0101_1111'],
@@ -360,12 +360,12 @@ class EmulativeTest extends LexerTest
             ]],
             ['function readonly(', [
                 [Tokens::T_FUNCTION, 'function'],
-                [Tokens::T_STRING, 'readonly'],
+                [Tokens::T_READONLY, 'readonly'],
                 [ord('('), '('],
             ]],
             ['function readonly (', [
                 [Tokens::T_FUNCTION, 'function'],
-                [Tokens::T_STRING, 'readonly'],
+                [Tokens::T_READONLY, 'readonly'],
                 [ord('('), '('],
             ]],
         ];

test/code/formatPreservation/comments.test

@@ -49,4 +49,4 @@ class Test {
     public function test() {
         // some code
     }
-}
+}

test/code/formatPreservation/inlineHtml.test

@@ -42,13 +42,71 @@ function test() {
     baz();
 }
 -----
-// TODO Fix broken result
+// TODO Preserve formatting
 $stmts[0]->stmts[1] = $stmts[0]->stmts[2];
 -----
 <?php
 
+function test()
+{
+    foo();
+    baz();
+    baz();
+}
+-----
+<?php
+
 function test() {
-    foo();<?php
+    foo();
+    ?>Bar<?php
     baz();
+}
+-----
+// TODO Preserve formatting
+unset($stmts[0]->stmts[2]);
+-----
+<?php
+
+function test()
+{
+    foo();
+    ?>Bar<?php
+}
+-----
+<?php
+
+function test() {
+    foo();
+    ?>Bar<?php
     baz();
-}
+}
+-----
+// TODO Preserve formatting
+array_splice($stmts[0]->stmts, 0, 1, []);
+-----
+<?php
+
+function test()
+{
+    ?>Bar<?php
+    baz();
+}
+-----
+<?php
+
+function test() {
+    foo();
+    ?>Bar<?php
+    baz();
+}
+-----
+// TODO Preserve formatting
+array_splice($stmts[0]->stmts, 1, 1, []);
+-----
+<?php
+
+function test()
+{
+    foo();
+    baz();
+}

test/code/parser/scalar/float.test

@@ -17,6 +17,7 @@ Different float syntaxes
 // (all are actually the same number, just in different representations)
 18446744073709551615;
 0xFFFFFFFFFFFFFFFF;
+0xEEEEEEEEEEEEEEEE;
 01777777777777777777777;
 0177777777777777777777787;
 0b1111111111111111111111111111111111111111111111111111111111111111;
@@ -92,7 +93,7 @@ array(
     )
     12: Stmt_Expression(
         expr: Scalar_DNumber(
-            value: 1.844674407371E+19
+            value: 1.7216961135462E+19
         )
     )
     13: Stmt_Expression(
@@ -105,4 +106,9 @@ array(
             value: 1.844674407371E+19
         )
     )
-)
+    15: Stmt_Expression(
+        expr: Scalar_DNumber(
+            value: 1.844674407371E+19
+        )
+    )
+)

test/code/parser/scalar/numberSeparators.test

@@ -4,7 +4,7 @@ Different integer syntaxes
 
 6.674_083e-11;
 299_792_458;
-0xCAFE_F00D;
+0x7AFE_F00D;
 0b0101_1111;
 0137_041;
 
@@ -42,7 +42,7 @@ array(
     )
     2: Stmt_Expression(
         expr: Scalar_LNumber(
-            value: 3405705229
+            value: 2063527949
         )
     )
     3: Stmt_Expression(
@@ -196,4 +196,4 @@ array(
             )
         )
     )
-)
+)

test/code/parser/stmt/class/modifier.test

@@ -280,7 +280,7 @@ array(
 // Type in the partial parse could conceivably be any of 0, 16 or 32
 -----
 !!php5
-Syntax error, unexpected T_READONLY from 1:7 to 1:14
+Syntax error, unexpected T_CLASS, expecting '(' from 1:16 to 1:20
 array(
     0: Stmt_Class(
         attrGroups: array(
@@ -366,4 +366,4 @@ array(
             )
         )
     )
-)
+)

test/code/parser/stmt/class/readonly.test

@@ -28,7 +28,7 @@ readonly class A {
 }
 -----
 !!php5
-Syntax error, unexpected T_READONLY from 3:1 to 3:8
+Syntax error, unexpected T_CLASS, expecting '(' from 3:10 to 3:14
 array(
     0: Stmt_Class(
         attrGroups: array(
@@ -65,4 +65,4 @@ array(
         stmts: array(
         )
     )
-)
+)

test/code/parser/stmt/function/disjointNormalFormTypes.test

@@ -4,6 +4,7 @@ DNF types
 
 class Test {
     public (A&B)|(X&Y) $prop;
+    public readonly (A&B)|C $prop2;
 }
 
 function test((A&B)|(X&Y) $a): (A&B)|(X&Y) {}
@@ -66,6 +67,42 @@ array(
                     )
                 )
             )
+            1: Stmt_Property(
+                attrGroups: array(
+                )
+                flags: MODIFIER_PUBLIC | MODIFIER_READONLY (65)
+                type: UnionType(
+                    types: array(
+                        0: IntersectionType(
+                            types: array(
+                                0: Name(
+                                    parts: array(
+                                        0: A
+                                    )
+                                )
+                                1: Name(
+                                    parts: array(
+                                        0: B
+                                    )
+                                )
+                            )
+                        )
+                        1: Name(
+                            parts: array(
+                                0: C
+                            )
+                        )
+                    )
+                )
+                props: array(
+                    0: Stmt_PropertyProperty(
+                        name: VarLikeIdentifier(
+                            name: prop2
+                        )
+                        default: null
+                    )
+                )
+            )
         )
     )
     1: Stmt_Function(

test/code/parser/stmt/function/readonlyFunction.test

@@ -0,0 +1,32 @@
+readonly function
+-----
+<?php
+function readonly() {}
+readonly();
+-----
+array(
+    0: Stmt_Function(
+        attrGroups: array(
+        )
+        byRef: false
+        name: Identifier(
+            name: readonly
+        )
+        params: array(
+        )
+        returnType: null
+        stmts: array(
+        )
+    )
+    1: Stmt_Expression(
+        expr: Expr_FuncCall(
+            name: Name(
+                parts: array(
+                    0: readonly
+                )
+            )
+            args: array(
+            )
+        )
+    )
+)

test/code/prettyPrinter/comments.test

@@ -64,4 +64,28 @@ function test()
 function test()
 {
     // empty
-}
+}
+-----
+<?php
+
+function noDuplicateComment()
+{
+    if (true):
+        // TEST 1
+    elseif (true):
+        // TEST 2
+    else:
+        // TEST 3
+    endif;
+}
+-----
+function noDuplicateComment()
+{
+    if (true) {
+        // TEST 1
+    } elseif (true) {
+        // TEST 2
+    } else {
+        // TEST 3
+    }
+}