Release PHP-Parser 5.0.2

Fixes #982.

.gitattributes

@@ -1,12 +1,17 @@
 /.github         export-ignore
 /doc             export-ignore
+/grammar         export-ignore
 /test            export-ignore
 /test_old        export-ignore
 /tools           export-ignore
 .editorconfig    export-ignore
 .gitattributes   export-ignore
 .gitignore       export-ignore
+.php-cs-fixer.dist.php  export-ignore
+Makefile         export-ignore
 CHANGELOG.md     export-ignore
 CONTRIBUTING.md  export-ignore
+phpstan-baseline.neon   export-ignore
+phpstan.neon.dist   export-ignore
 phpunit.xml.dist export-ignore
 UPGRADE-*.md     export-ignore

.github/workflows/main.yml

@@ -116,4 +116,4 @@ jobs:
         run: |
           cd tools && composer install
       - name: "php-cs-fixer"
-        run: "php tools/vendor/bin/php-cs-fixer fix"
+        run: "php tools/vendor/bin/php-cs-fixer fix --dry-run"

.php-cs-fixer.dist.php

@@ -21,11 +21,6 @@ return $config->setRiskyAllowed(true)
         'declare_strict_types' => true,
         // Keep argument formatting for now.
         'method_argument_space' => ['on_multiline' => 'ignore'],
-        'binary_operator_spaces' => [
-            'default' => 'at_least_single_space',
-            // Work around https://github.com/PHP-CS-Fixer/PHP-CS-Fixer/issues/7303.
-            'operators' => ['=' => null],
-        ],
         'phpdoc_align' => ['align' => 'left'],
         'phpdoc_trim' => true,
         'no_empty_phpdoc' => true,

CHANGELOG.md

@@ -1,3 +1,67 @@
+Version 5.0.2 (2024-03-05)
+--------------------------
+
+### Fixed
+
+* Fix handling of indentation on next line after opening PHP tag in formatting-preserving pretty
+printer.
+
+### Changed
+
+* Avoid cyclic references in `Parser` objects. This means that no longer used parser objects are
+  immediately destroyed now, instead of requiring cycle GC.
+* Update `PhpVersion::getNewestSupported()` to report PHP 8.3 instead of PHP 8.2.
+
+Version 5.0.1 (2024-02-21)
+--------------------------
+
+### Changed
+
+* Added check to detect use of PHP-Parser with libraries that define `T_*` compatibility tokens
+  with incorrect type (such as string instead of int). This would lead to `TypeError`s down the
+  line. Now an `Error` will be thrown early to indicate the problem.
+
+Version 5.0.0 (2024-01-07)
+--------------------------
+
+See UPGRADE-5.0 for detailed migration instructions.
+
+### Fixed
+
+* Fixed parent class of `PropertyItem` and `UseItem`.
+
+Version 5.0.0-rc1 (2023-12-20)
+------------------------------
+
+See UPGRADE-5.0 for detailed migration instructions.
+
+### Fixed
+
+* Fixed parsing of empty files.
+
+### Added
+
+* Added support for printing additional attributes (like `kind`) in `NodeDumper`.
+* Added `rawValue` attribute to `InterpolatedStringPart` and heredoc/nowdoc `String_`s, which
+  provides the original, unparsed value. It was previously only available for non-interpolated
+  single/double quoted strings.
+* Added `Stmt\Block` to represent `{}` code blocks. Previously, such code blocks were flattened
+  into the parent statements array. `Stmt\Block` will not be created for structures that are
+  typically used with code blocks, for example `if ($x) { $y; }` will be represented as previously,
+  while `if ($x) { { $x; } }` will have an extra `Stmt\Block` wrapper.
+
+### Changed
+
+* Use visitor to assign comments. This fixes the long-standing issue where comments were assigned
+  to all nodes sharing a starting position. Now only the outer-most node will hold the comments.
+* Don't parse unicode escape sequences when targeting PHP < 7.0.
+* Improve NodeDumper performance for large dumps.
+
+### Removed
+
+* Removed `Stmt\Throw_` node, use `Expr\Throw_` inside `Stmt\Expression` instead.
+* Removed `ParserFactory::create()`.
+
 Version 5.0.0-beta1 (2023-09-17)
 --------------------------------
 

CONTRIBUTING.md

@@ -2,3 +2,31 @@
 
 This project uses PSR-12 with consistent brace placement. This means that the opening brace is
 always on the same line, even for class and method declarations.
+
+## Tools
+
+This project uses PHP-CS-Fixer and PHPStan. You can invoke them using `make`:
+
+```shell
+make php-cs-fixer
+make phpstan
+```
+
+## Adding support for new PHP syntax
+
+1. If necessary, add emulation support for new tokens.
+   * Add a new subclass of `Lexer\TokenEmulator`. Take inspiration from existing classes.
+   * Add the new class to the array in `Lexer\Emulative`.
+   * Add tests for the emulation in `Lexer\EmulativeTest`. You'll want to modify
+     `provideTestReplaceKeywords()` for new reserved keywords and `provideTestLexNewFeatures()` for
+     other emulations.
+2. Add any new node classes that are needed.
+3. Add support for the new syntax in `grammar/php.y`. Regenerate the parser by running
+   `php grammar/rebuildParsers.php`. Use `--debug` if there are conflicts.
+4. Add pretty-printing support by implementing a `pFooBar()` method in `PrettyPrinter\Standard`.
+5. Add tests both in `test/code/parser` and `test/code/prettyPrinter`.
+6. Add support for formatting-preserving pretty-printing. This is done by modifying the data tables
+   at the end of `PrettyPrinterAbstract`. Add a test in `test/code/formatPreservation`.
+7. Does the new syntax feature namespaced names? If so, add support for name resolution in
+   `NodeVisitor\NameResolver`. Test it in `NodeVisitor\NameResolverTest`.
+8. Does the new syntax require any changes to builders? Is so, make them :)

README.md

@@ -6,11 +6,9 @@ PHP Parser
 This is a PHP parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
-[Documentation for version 5.x][doc_master] (in development; for running on PHP >= 7.4; for parsing PHP 7.0 to PHP 8.3, with limited support for parsing PHP 5.x).
+[**Documentation for version 5.x**][doc_master] (current; for running on PHP >= 7.4; for parsing PHP 7.0 to PHP 8.3, with limited support for parsing PHP 5.x).
 
-[**Documentation for version 4.x**][doc_4_x] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 8.3).
-
-[Documentation for version 3.x][doc_3_x] (unsupported; for running on PHP >= 5.5; for parsing PHP 5.2 to PHP 7.2).
+[Documentation for version 4.x][doc_4_x] (supported; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 8.3).
 
 Features
 --------
@@ -213,9 +211,8 @@ Component documentation:
  * [AST builders](doc/component/AST_builders.markdown)
    * Fluent builders for AST nodes
  * [Lexer](doc/component/Lexer.markdown)
-   * Lexer options
-   * Token and file positions for nodes
-   * Custom attributes
+   * Emulation
+   * Tokens, positions and attributes
  * [Error handling](doc/component/Error_handling.markdown)
    * Column information for errors
    * Error recovery (parsing of syntactically incorrect code)

UPGRADE-5.0.md

@@ -13,6 +13,7 @@ In particular, if an older `PhpVersion` is specified, then:
 
  * For versions before PHP 7.0, `$foo =& new Bar()` assignments are allowed without error.
  * For versions before PHP 7.0, invalid octal literals `089` are allowed without error.
+ * For versions before PHP 7.0, unicode escape sequences `\u{123}` in strings are not parsed.
  * Type hints are interpreted as a class `Name` or as a built-in `Identifier` depending on PHP
    version, for example `int` is treated as a class name on PHP 5.6 and as a built-in on PHP 7.0.
 
@@ -27,16 +28,17 @@ The following symbols are affected by this removal:
  * The `PhpParser\Parser\Php5` class has been removed.
  * The `PhpParser\Parser\Multiple` class has been removed. While not strictly related to PHP 5 support, this functionality is no longer useful without it.
  * The `PhpParser\ParserFactory::ONLY_PHP5` and `PREFER_PHP5` options have been removed.
- * The `PhpParser\ParserFactory::PREFER_PHP7` option is now equivalent to `ONLY_PHP7`.
 
 ### Changes to the parser factory
 
-The `ParserFactory::create()` method is deprecated in favor of three new methods that provide more fine-grained control over the PHP version being targeted:
+The `ParserFactory::create()` method has been removed in favor of three new methods that provide more fine-grained control over the PHP version being targeted:
 
  * `createForNewestSupportedVersion()`: Use this if you don't know the PHP version of the code you're parsing. It's better to assume a too new version than a too old one.
  * `createForHostVersion()`: Use this if you're parsing code for the PHP version you're running on.
  * `createForVersion()`: Use this if you know the PHP version of the code you want to parse.
 
+The `createForNewestSupportedVersion()` and `creatForHostVersion()` are available since PHP-Parser 4.18.0, to allow libraries to support PHP-Parser 4 and 5 at the same time more easily.
+
 In all cases, the PHP version is a fairly weak hint that is only used on a best-effort basis. The parser will usually accept code for newer versions if it does not have any backwards-compatibility implications.
 
 For example, if you specify version `"8.0"`, then `class ReadOnly {}` is treated as a valid class declaration, while using `public readonly int $prop` will lead to a parse error. However, `final public const X = Y;` will be accepted in both cases.
@@ -61,6 +63,35 @@ $parser = $factory->create(ParserFactory::ONLY_PHP5);
 $parser = $factory->createForVersion(PhpVersion::fromString("5.6"));
 ```
 
+### Changes to the throw representation
+
+Previously, `throw` statements like `throw $e;` were represented using the `Stmt\Throw_` class,
+while uses inside other expressions (such as `$x ?? throw $e`) used the `Expr\Throw_` class.
+
+Now, `throw $e;` is represented as a `Stmt\Expression` that contains an `Expr\Throw_`. The
+`Stmt\Throw_` class has been removed.
+
+```php
+# Code
+throw $e;
+
+# Before
+Stmt_Throw(
+    expr: Expr_Variable(
+        name: e
+    )
+)
+
+# After
+Stmt_Expression(
+    expr: Expr_Throw(
+        expr: Expr_Variable(
+            name: e
+        )
+    )
+)
+```
+
 ### Changes to the array destructuring representation
 
 Previously, the `list($x) = $y` destructuring syntax was represented using a `Node\Expr\List_`
@@ -71,6 +102,49 @@ Now, destructuring is always represented using `Node\Expr\List_`. The `kind` att
 `Node\Expr\List_::KIND_LIST` or `Node\Expr\List_::KIND_ARRAY` specifies which syntax was actually
 used.
 
+```php
+# Code
+[$x] = $y;
+
+# Before
+Expr_Assign(
+   var: Expr_Array(
+       items: array(
+           0: Expr_ArrayItem(
+               key: null
+               value: Expr_Variable(
+                   name: x
+               )
+               byRef: false
+               unpack: false
+           )
+       )
+   )
+   expr: Expr_Variable(
+       name: y
+   )
+)
+
+# After
+Expr_Assign(
+   var: Expr_List(
+       items: array(
+           0: ArrayItem(
+               key: null
+               value: Expr_Variable(
+                   name: x
+               )
+               byRef: false
+               unpack: false
+           )
+       )
+   )
+   expr: Expr_Variable(
+       name: y
+   )
+)
+```
+
 ### Changes to the name representation
 
 Previously, `Name` nodes had a `parts` subnode, which stores an array of name parts, split by
@@ -82,6 +156,123 @@ now represented by `Name(name: 'Foo\Bar')` instead.
 It is possible to convert the name to the previous representation using `$name->getParts()`. The
 `Name` constructor continues to accept both the string and the array representation.
 
+The `Name::getParts()` method is available since PHP-Parser 4.16.0, to allow libraries to support
+PHP-Parser 4 and 5 at the same time more easily.
+
+### Changes to the block representation
+
+Previously, code blocks `{ ... }` were always flattened into their parent statement list. For
+example `while ($x) { $a; { $b; } $c; }` would produce the same node structure as
+`if ($x) { $a; $b; $c; }`, namely a `Stmt\While_` node whose `stmts` subnode is an array of three
+statements.
+
+Now, the nested `{ $b; }` block is represented using an explicit `Stmt\Block` node. However, the
+outer `{ $a; { $b; } $c; }` block is still represented using a simple array in the `stmts` subnode.
+
+```php
+# Code
+while ($x) { $a; { $b; } $c; }
+
+# Before
+Stmt_While(
+    cond: Expr_Variable(
+        name: x
+    )
+    stmts: array(
+        0: Stmt_Expression(
+            expr: Expr_Variable(
+                name: a
+            )
+        )
+        1: Stmt_Expression(
+            expr: Expr_Variable(
+                name: b
+            )
+        )
+        2: Stmt_Expression(
+            expr: Expr_Variable(
+                name: c
+            )
+        )
+    )
+)
+
+# After
+Stmt_While(
+    cond: Expr_Variable(
+        name: x
+    )
+    stmts: array(
+        0: Stmt_Expression(
+            expr: Expr_Variable(
+                name: a
+            )
+        )
+        1: Stmt_Block(
+            stmts: array(
+                0: Stmt_Expression(
+                    expr: Expr_Variable(
+                        name: b
+                    )
+                )
+            )
+        )
+        2: Stmt_Expression(
+            expr: Expr_Variable(
+                name: c
+            )
+        )
+    )
+)
+```
+
+### Changes to comment assignment
+
+Previously, comments were assigned to all nodes starting at the same position. Now they will be
+assigned to the outermost node only.
+
+```php
+# Code
+// Comment
+$a + $b;
+
+# Before
+Stmt_Expression(
+    expr: Expr_BinaryOp_Plus(
+        left: Expr_Variable(
+            name: a
+            comments: array(
+                0: // Comment
+            )
+        )
+        right: Expr_Variable(
+            name: b
+        )
+        comments: array(
+            0: // Comment
+        )
+    )
+    comments: array(
+        0: // Comment
+    )
+)
+
+# After
+Stmt_Expression(
+    expr: Expr_BinaryOp_Plus(
+        left: Expr_Variable(
+            name: a
+        )
+        right: Expr_Variable(
+            name: b
+        )
+    )
+    comments: array(
+        0: // Comment
+    )
+)
+```
+
 ### Renamed nodes
 
 A number of AST nodes have been renamed or moved in the AST hierarchy:
@@ -104,7 +295,7 @@ The old class names have been retained as aliases for backwards compatibility. H
 
 Modifier flags (as used by the `$flags` subnode of `Class_`, `ClassMethod`, `Property`, etc.) are now available as class constants on a separate `PhpParser\Modifiers` class, instead of being part of `PhpParser\Node\Stmt\Class_`, to make it clearer that these are used by many different nodes. The old constants are deprecated, but are still available.
 
-```
+```php
 PhpParser\Node\Stmt\Class_::MODIFIER_PUBLIC    -> PhpParser\Modifiers::PUBLIC
 PhpParser\Node\Stmt\Class_::MODIFIER_PROTECTED -> PhpParser\Modifiers::PROTECTED
 PhpParser\Node\Stmt\Class_::MODIFIER_PRIVATE   -> PhpParser\Modifiers::PRIVATE
@@ -202,7 +393,7 @@ The pretty printer now accepts a `phpVersion` option, which accepts a `PhpVersio
 
 * For PHP >= 7.0 (default), short array syntax `[]` will be used by default. This does not affect nodes that specify an explicit array syntax using the `kind` attribute.
 * For PHP >= 7.0 (default), parentheses around `yield` expressions will only be printed when necessary. Previously, parentheses were always printed, even if `yield` was used as a statement.
-* For PHP >= 7.1 (default), the short array syntax `[]` will be used for destructuring by default (instead of `list()`). This does not affect nodes that specify and explicit syntax using the `kind` attribute.
+* For PHP >= 7.1 (default), the short array syntax `[]` will be used for destructuring by default (instead of `list()`). This does not affect nodes that specify an explicit syntax using the `kind` attribute.
 * For PHP >= 7.3 (default), a newline is no longer forced after heredoc/nowdoc strings, as the requirement for this has been removed with the introduction of flexible heredoc/nowdoc strings.
 * For PHP >= 7.3 (default), heredoc/nowdoc strings are now indented just like regular code. This was allowed with the introduction of flexible heredoc/nowdoc strings.
 
@@ -220,7 +411,7 @@ protected function p(
 
 The `$precedence` is the precedence of the direct parent operator (if any), while `$lhsPrecedence` is that precedence of the nearest binary operator on whose left-hand-side the node occurs. For unary operators, only the `$lhsPrecedence` is relevant.
 
-Recursive calls in pretty-printer methods should generally continue calling `p()` without additional parameters. However, pretty-printer methods for operators that participate in precedence resolution need to be adjusted. For example, typical implementations for operators looks as follows now:
+Recursive calls in pretty-printer methods should generally continue calling `p()` without additional parameters. However, pretty-printer methods for operators that participate in precedence resolution need to be adjusted. For example, typical implementations for operators look as follows now:
 
 ```php
 protected function pExpr_BinaryOp_Plus(
@@ -301,7 +492,7 @@ Additionally, the token array is now terminated by a sentinel token with ID 0.
 
 The lexer API is reduced to a single `Lexer::tokenize()` method, which returns an array of tokens. The `startLexing()` and `getNextToken()` methods have been removed.
 
-Responsibility for determining start and end attributes for nodes has been moved from the lexer to the parser. The lexer no longer accepts an options array. The `usedAttributes` option has been removed without replacement, and the parser will now unconditionally add the `comments`, `startLine`, `endLine`, `startFilePos`, `startEndPos`, `startTokenPos` and `startEndPos` attributes.
+Responsibility for determining start and end attributes for nodes has been moved from the lexer to the parser. The lexer no longer accepts an options array. The `usedAttributes` option has been removed without replacement, and the parser will now unconditionally add the `comments`, `startLine`, `endLine`, `startFilePos`, `endFilePos`, `startTokenPos` and `endTokenPos` attributes.
 
 There should no longer be a need to directly interact with the `Lexer` for end users, as the `ParserFactory` will create an appropriate instance, and no additional configuration of the lexer is necessary. To use formatting-preserving pretty printing, the setup boilerplate changes as follows:
 
@@ -328,7 +519,7 @@ $newStmts = $traverser->traverse($oldStmts);
 
 $parser = (new ParserFactory())->createForNewestSupportedVersion();
 $oldStmts = $parser->parse($code);
-$oldTokens = $lexer->getTokens();
+$oldTokens = $parser->getTokens();
 
 $traverser = new NodeTraverser(new NodeVisitor\CloningVisitor());
 $newStmts = $traverser->traverse($oldStmts);

doc/README.md

@@ -27,9 +27,8 @@ Component documentation
   * [AST builders](component/AST_builders.markdown)
     * Fluent builders for AST nodes
   * [Lexer](component/Lexer.markdown)
-    * Lexer options
-    * Token and file positions for nodes
-    * Custom attributes
+    * Emulation
+    * Tokens, positions and attributes
   * [Error handling](component/Error_handling.markdown)
     * Column information for errors
     * Error recovery (parsing of syntactically incorrect code)

doc/component/Pretty_printing.markdown

@@ -97,5 +97,5 @@ If you make use of the name resolution functionality, you will likely want to di
 the AST and causing spurious changes to the pretty printed code. For more information, see the
 [name resolution documentation](Name_resolution.markdown).
 
-The formatting-preservation works on a best-effort basis and may sometimes reformat more code tha
+The formatting-preservation works on a best-effort basis and may sometimes reformat more code than
 necessary. If you encounter problems while using this functionality, please open an issue.

grammar/parser.template

@@ -1,7 +1,7 @@
 <?php declare(strict_types=1);
 $meta #
-#semval($) $this->semValue
-#semval($,%t) $this->semValue
+#semval($) $self->semValue
+#semval($,%t) $self->semValue
 #semval(%n) $stackPos-(%l-%n)
 #semval(%n,%t) $stackPos-(%l-%n)
 
@@ -28,68 +28,68 @@ class #(-p) extends \PhpParser\ParserAbstract
     public const %s = %n;
 #endtokenval
 
-    protected $tokenToSymbolMapSize = #(YYMAXLEX);
-    protected $actionTableSize = #(YYLAST);
-    protected $gotoTableSize = #(YYGLAST);
+    protected int $tokenToSymbolMapSize = #(YYMAXLEX);
+    protected int $actionTableSize = #(YYLAST);
+    protected int $gotoTableSize = #(YYGLAST);
 
-    protected $invalidSymbol = #(YYBADCH);
-    protected $errorSymbol = #(YYINTERRTOK);
-    protected $defaultAction = #(YYDEFAULT);
-    protected $unexpectedTokenRule = #(YYUNEXPECTED);
+    protected int $invalidSymbol = #(YYBADCH);
+    protected int $errorSymbol = #(YYINTERRTOK);
+    protected int $defaultAction = #(YYDEFAULT);
+    protected int $unexpectedTokenRule = #(YYUNEXPECTED);
 
-    protected $YY2TBLSTATE = #(YY2TBLSTATE);
-    protected $numNonLeafStates = #(YYNLSTATES);
+    protected int $YY2TBLSTATE = #(YY2TBLSTATE);
+    protected int $numNonLeafStates = #(YYNLSTATES);
 
-    protected $symbolToName = array(
+    protected array $symbolToName = array(
         #listvar terminals
     );
 
-    protected $tokenToSymbol = array(
+    protected array $tokenToSymbol = array(
         #listvar yytranslate
     );
 
-    protected $action = array(
+    protected array $action = array(
         #listvar yyaction
     );
 
-    protected $actionCheck = array(
+    protected array $actionCheck = array(
         #listvar yycheck
     );
 
-    protected $actionBase = array(
+    protected array $actionBase = array(
         #listvar yybase
     );
 
-    protected $actionDefault = array(
+    protected array $actionDefault = array(
         #listvar yydefault
     );
 
-    protected $goto = array(
+    protected array $goto = array(
         #listvar yygoto
     );
 
-    protected $gotoCheck = array(
+    protected array $gotoCheck = array(
         #listvar yygcheck
     );
 
-    protected $gotoBase = array(
+    protected array $gotoBase = array(
         #listvar yygbase
     );
 
-    protected $gotoDefault = array(
+    protected array $gotoDefault = array(
         #listvar yygdefault
     );
 
-    protected $ruleToNonTerminal = array(
+    protected array $ruleToNonTerminal = array(
         #listvar yylhs
     );
 
-    protected $ruleToLength = array(
+    protected array $ruleToLength = array(
         #listvar yylen
     );
 #if -t
 
-    protected $productions = array(
+    protected array $productions = array(
         #production-strings;
     );
 #endif
@@ -97,7 +97,7 @@ class #(-p) extends \PhpParser\ParserAbstract
     protected function initReduceCallbacks(): void {
         $this->reduceCallbacks = [
 #reduce
-            %n => function ($stackPos) {
+            %n => static function ($self, $stackPos) {
                 %b
             },
 #noact

grammar/php.y

@@ -366,21 +366,13 @@ inner_statement:
 ;
 
 non_empty_statement:
-      '{' inner_statement_list '}'
-    {
-        if ($2) {
-            $$ = $2; prependLeadingComments($$);
-        } else {
-            makeNop($$);
-            if (null === $$) { $$ = array(); }
-        }
-    }
-    | T_IF '(' expr ')' statement elseif_list else_single
-          { $$ = Stmt\If_[$3, ['stmts' => toArray($5), 'elseifs' => $6, 'else' => $7]]; }
+      '{' inner_statement_list '}'                          { $$ = Stmt\Block[$2]; }
+    | T_IF '(' expr ')' blocklike_statement elseif_list else_single
+          { $$ = Stmt\If_[$3, ['stmts' => $5, 'elseifs' => $6, 'else' => $7]]; }
     | T_IF '(' expr ')' ':' inner_statement_list new_elseif_list new_else_single T_ENDIF ';'
           { $$ = Stmt\If_[$3, ['stmts' => $6, 'elseifs' => $7, 'else' => $8]]; }
     | T_WHILE '(' expr ')' while_statement                  { $$ = Stmt\While_[$3, $5]; }
-    | T_DO statement T_WHILE '(' expr ')' ';'               { $$ = Stmt\Do_   [$5, toArray($2)]; }
+    | T_DO blocklike_statement T_WHILE '(' expr ')' ';'     { $$ = Stmt\Do_   [$5, $2]; }
     | T_FOR '(' for_expr ';'  for_expr ';' for_expr ')' for_statement
           { $$ = Stmt\For_[['init' => $3, 'cond' => $5, 'loop' => $7, 'stmts' => $9]]; }
     | T_SWITCH '(' expr ')' switch_case_list                { $$ = Stmt\Switch_[$3, $5]; }
@@ -394,16 +386,7 @@ non_empty_statement:
         $$ = Stmt\InlineHTML[$1];
         $$->setAttribute('hasLeadingNewline', $this->inlineHtmlHasLeadingNewline(#1));
     }
-    | expr semi {
-        $e = $1;
-        if ($e instanceof Expr\Throw_) {
-            // For backwards-compatibility reasons, convert throw in statement position into
-            // Stmt\Throw_ rather than Stmt\Expression(Expr\Throw_).
-            $$ = Stmt\Throw_[$e->expr];
-        } else {
-            $$ = Stmt\Expression[$e];
-        }
-    }
+    | expr semi                                             { $$ = Stmt\Expression[$1]; }
     | T_UNSET '(' variables_list ')' semi                   { $$ = Stmt\Unset_[$3]; }
     | T_FOREACH '(' expr T_AS foreach_variable ')' foreach_statement
           { $$ = Stmt\Foreach_[$3, $5[0], ['keyVar' => null, 'byRef' => $5[1], 'stmts' => $7]]; }
@@ -416,14 +399,16 @@ non_empty_statement:
           { $$ = Stmt\TryCatch[$3, $5, $6]; $this->checkTryCatch($$); }
     | T_GOTO identifier_not_reserved semi                   { $$ = Stmt\Goto_[$2]; }
     | identifier_not_reserved ':'                           { $$ = Stmt\Label[$1]; }
-    | error                                                 { $$ = array(); /* means: no statement */ }
+    | error                                                 { $$ = null; /* means: no statement */ }
 ;
 
 statement:
       non_empty_statement
-    | ';'
-          { makeNop($$);
-            if ($$ === null) $$ = array(); /* means: no statement */ }
+    | ';'                                                   { makeNop($$); }
+;
+
+blocklike_statement:
+     statement                                              { toBlock($1); }
 ;
 
 catches:
@@ -554,17 +539,17 @@ non_empty_class_name_list:
 ;
 
 for_statement:
-      statement                                             { $$ = toArray($1); }
+      blocklike_statement
     | ':' inner_statement_list T_ENDFOR ';'                 { $$ = $2; }
 ;
 
 foreach_statement:
-      statement                                             { $$ = toArray($1); }
+      blocklike_statement
     | ':' inner_statement_list T_ENDFOREACH ';'             { $$ = $2; }
 ;
 
 declare_statement:
-      non_empty_statement                                   { $$ = toArray($1); }
+      non_empty_statement                                   { toBlock($1); }
     | ';'                                                   { $$ = null; }
     | ':' inner_statement_list T_ENDDECLARE ';'             { $$ = $2; }
 ;
@@ -624,7 +609,7 @@ match_arm:
 ;
 
 while_statement:
-      statement                                             { $$ = toArray($1); }
+      blocklike_statement                                   { $$ = $1; }
     | ':' inner_statement_list T_ENDWHILE ';'               { $$ = $2; }
 ;
 
@@ -634,7 +619,7 @@ elseif_list:
 ;
 
 elseif:
-      T_ELSEIF '(' expr ')' statement                       { $$ = Stmt\ElseIf_[$3, toArray($5)]; }
+      T_ELSEIF '(' expr ')' blocklike_statement             { $$ = Stmt\ElseIf_[$3, $5]; }
 ;
 
 new_elseif_list:
@@ -649,7 +634,7 @@ new_elseif:
 
 else_single:
       /* empty */                                           { $$ = null; }
-    | T_ELSE statement                                      { $$ = Stmt\Else_[toArray($2)]; }
+    | T_ELSE blocklike_statement                            { $$ = Stmt\Else_[$2]; }
 ;
 
 new_else_single:
@@ -831,7 +816,7 @@ static_var:
 ;
 
 class_statement_list_ex:
-      class_statement_list_ex class_statement               { if ($2 !== null) { push($1, $2); } }
+      class_statement_list_ex class_statement               { if ($2 !== null) { push($1, $2); } else { $$ = $1; } }
     | /* empty */                                           { init(); }
 ;
 
@@ -1151,9 +1136,9 @@ exit_expr:
 
 backticks_expr:
       /* empty */                                           { $$ = array(); }
-    | T_ENCAPSED_AND_WHITESPACE
-          { $$ = array(Node\InterpolatedStringPart[Scalar\String_::parseEscapeSequences($1, '`')]); }
-    | encaps_list                                           { parseEncapsed($1, '`', true); $$ = $1; }
+    | encaps_string_part
+          { $$ = array($1); parseEncapsed($$, '`', $this->phpVersion->supportsUnicodeEscapes()); }
+    | encaps_list                                           { parseEncapsed($1, '`', $this->phpVersion->supportsUnicodeEscapes()); $$ = $1; }
 ;
 
 ctor_arguments:
@@ -1196,10 +1181,11 @@ dereferencable_scalar:
             $$ = new Expr\Array_($3, $attrs);
             $this->createdArrays->attach($$); }
     | array_short_syntax                                    { $$ = $1; $this->createdArrays->attach($$); }
-    | T_CONSTANT_ENCAPSED_STRING                            { $$ = Scalar\String_::fromString($1, attributes()); }
+    | T_CONSTANT_ENCAPSED_STRING
+          { $$ = Scalar\String_::fromString($1, attributes(), $this->phpVersion->supportsUnicodeEscapes()); }
     | '"' encaps_list '"'
           { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
-            parseEncapsed($2, '"', true); $$ = new Scalar\InterpolatedString($2, $attrs); }
+            parseEncapsed($2, '"', $this->phpVersion->supportsUnicodeEscapes()); $$ = new Scalar\InterpolatedString($2, $attrs); }
 ;
 
 scalar:
@@ -1352,7 +1338,8 @@ encaps_list:
 ;
 
 encaps_string_part:
-      T_ENCAPSED_AND_WHITESPACE                             { $$ = Node\InterpolatedStringPart[$1]; }
+      T_ENCAPSED_AND_WHITESPACE
+          { $attrs = attributes(); $attrs['rawValue'] = $1; $$ = new Node\InterpolatedStringPart($1, $attrs); }
 ;
 
 encaps_str_varname:

grammar/phpyLang.php

@@ -23,6 +23,7 @@ function preprocessGrammar($code) {
     $code = resolveNodes($code);
     $code = resolveMacros($code);
     $code = resolveStackAccess($code);
+    $code = str_replace('$this', '$self', $code);
 
     return $code;
 }
@@ -87,14 +88,15 @@ function resolveMacros($code) {
             if ('pushNormalizing' === $name) {
                 assertArgs(2, $args, $name);
 
-                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
-                       . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
+                return 'if (' . $args[1] . ' !== null) { ' . $args[0] . '[] = ' . $args[1] . '; } $$ = ' . $args[0] . ';';
             }
 
-            if ('toArray' == $name) {
+            if ('toBlock' == $name) {
                 assertArgs(1, $args, $name);
 
-                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
+                return 'if (' . $args[0] . ' instanceof Stmt\Block) { $$ = ' . $args[0] . '->stmts; } '
+                     . 'else if (' . $args[0] . ' === null) { $$ = []; } '
+                     . 'else { $$ = [' . $args[0] . ']; }';
             }
 
             if ('parseVar' === $name) {
@@ -122,15 +124,6 @@ function resolveMacros($code) {
                 return $args[0] . ' = $this->maybeCreateZeroLengthNop($this->tokenPos);';
             }
 
-            if ('prependLeadingComments' === $name) {
-                assertArgs(1, $args, $name);
-
-                return '$comments = $this->getCommentsBeforeToken($this->tokenStartStack[#1]); $stmts = ' . $args[0] . '; '
-                       . 'if (!empty($comments)) {'
-                       . '$stmts[0]->setAttribute(\'comments\', '
-                       . 'array_merge($comments, $stmts[0]->getAttribute(\'comments\', []))); }';
-            }
-
             return $matches[0];
         },
         $code

lib/PhpParser/Internal/TokenStream.php

@@ -195,12 +195,6 @@ class TokenStream {
         return false;
     }
 
-    public function haveBracesInRange(int $startPos, int $endPos): bool {
-        return $this->haveTokenInRange($startPos, $endPos, '{')
-            || $this->haveTokenInRange($startPos, $endPos, T_CURLY_OPEN)
-            || $this->haveTokenInRange($startPos, $endPos, '}');
-    }
-
     public function haveTagInRange(int $startPos, int $endPos): bool {
         return $this->haveTokenInRange($startPos, $endPos, \T_OPEN_TAG)
             || $this->haveTokenInRange($startPos, $endPos, \T_CLOSE_TAG);
@@ -257,7 +251,7 @@ class TokenStream {
     private function calcIndentMap(): array {
         $indentMap = [];
         $indent = 0;
-        foreach ($this->tokens as $token) {
+        foreach ($this->tokens as $i => $token) {
             $indentMap[] = $indent;
 
             if ($token->id === \T_WHITESPACE) {
@@ -265,6 +259,10 @@ class TokenStream {
                 $newlinePos = \strrpos($content, "\n");
                 if (false !== $newlinePos) {
                     $indent = \strlen($content) - $newlinePos - 1;
+                } elseif ($i === 1 && $this->tokens[0]->id === \T_OPEN_TAG &&
+                          $this->tokens[0]->text[\strlen($this->tokens[0]->text) - 1] === "\n") {
+                    // Special case: Newline at the end of opening tag followed by whitespace.
+                    $indent = \strlen($content);
                 }
             }
         }

lib/PhpParser/Lexer.php

@@ -76,7 +76,7 @@ class Lexer {
         $numTokens = \count($tokens);
         if ($numTokens === 0) {
             // Empty input edge case: Just add the sentinel token.
-            $tokens[] = [new Token(0, "\0", 1, 0)];
+            $tokens[] = new Token(0, "\0", 1, 0);
             return;
         }
 

lib/PhpParser/Node/Expr/ArrowFunction.php

@@ -7,8 +7,10 @@ use PhpParser\Node\Expr;
 use PhpParser\Node\FunctionLike;
 
 class ArrowFunction extends Expr implements FunctionLike {
+    /** @var bool Whether the closure is static */
     public bool $static;
 
+    /** @var bool Whether to return by reference */
     public bool $byRef;
 
     /** @var Node\Param[] */
@@ -17,6 +19,7 @@ class ArrowFunction extends Expr implements FunctionLike {
     /** @var null|Node\Identifier|Node\Name|Node\ComplexType */
     public ?Node $returnType;
 
+    /** @var Expr Expression body */
     public Expr $expr;
     /** @var Node\AttributeGroup[] */
     public array $attrGroups;

lib/PhpParser/Node/Expr/Match_.php

@@ -6,12 +6,15 @@ use PhpParser\Node;
 use PhpParser\Node\MatchArm;
 
 class Match_ extends Node\Expr {
+    /** @var Node\Expr Condition */
     public Node\Expr $cond;
     /** @var MatchArm[] */
     public array $arms;
 
     /**
+     * @param Node\Expr $cond Condition
      * @param MatchArm[] $arms
+     * @param array<string, mixed> $attributes Additional attributes
      */
     public function __construct(Node\Expr $cond, array $arms = [], array $attributes = []) {
         $this->attributes = $attributes;

lib/PhpParser/Node/Param.php

@@ -17,6 +17,7 @@ class Param extends NodeAbstract {
     public Expr $var;
     /** @var null|Expr Default value */
     public ?Expr $default;
+    /** @var int Optional visibility flags */
     public int $flags;
     /** @var AttributeGroup[] PHP attribute groups */
     public array $attrGroups;

lib/PhpParser/Node/PropertyItem.php

@@ -3,8 +3,9 @@
 namespace PhpParser\Node;
 
 use PhpParser\Node;
+use PhpParser\NodeAbstract;
 
-class PropertyItem extends Node\Stmt {
+class PropertyItem extends NodeAbstract {
     /** @var Node\VarLikeIdentifier Name */
     public VarLikeIdentifier $name;
     /** @var null|Node\Expr Default */

lib/PhpParser/Node/Stmt/Block.php

@@ -0,0 +1,29 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node\Stmt;
+
+use PhpParser\Node\Stmt;
+
+class Block extends Stmt {
+    /** @var Stmt[] Statements */
+    public array $stmts;
+
+    /**
+     * A block of statements.
+     *
+     * @param Stmt[] $stmts Statements
+     * @param array<string, mixed> $attributes Additional attributes
+     */
+    public function __construct(array $stmts, array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->stmts = $stmts;
+    }
+
+    public function getType(): string {
+        return 'Stmt_Block';
+    }
+
+    public function getSubNodeNames(): array {
+        return ['stmts'];
+    }
+}

lib/PhpParser/Node/Stmt/Throw_.php

@@ -1,29 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node\Stmt;
-
-use PhpParser\Node;
-
-class Throw_ extends Node\Stmt {
-    /** @var Node\Expr Expression */
-    public Node\Expr $expr;
-
-    /**
-     * Constructs a legacy throw statement node.
-     *
-     * @param Node\Expr $expr Expression
-     * @param array<string, mixed> $attributes Additional attributes
-     */
-    public function __construct(Node\Expr $expr, array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->expr = $expr;
-    }
-
-    public function getSubNodeNames(): array {
-        return ['expr'];
-    }
-
-    public function getType(): string {
-        return 'Stmt_Throw';
-    }
-}

lib/PhpParser/Node/UseItem.php

@@ -3,9 +3,10 @@
 namespace PhpParser\Node;
 
 use PhpParser\Node;
+use PhpParser\NodeAbstract;
 use PhpParser\Node\Stmt\Use_;
 
-class UseItem extends Node\Stmt {
+class UseItem extends NodeAbstract {
     /**
      * @var Use_::TYPE_* One of the Stmt\Use_::TYPE_* constants. Will only differ from TYPE_UNKNOWN for mixed group uses
      */

lib/PhpParser/NodeDumper.php

@@ -2,7 +2,12 @@
 
 namespace PhpParser;
 
+use PhpParser\Node\Expr\Array_;
 use PhpParser\Node\Expr\Include_;
+use PhpParser\Node\Expr\List_;
+use PhpParser\Node\Scalar\Int_;
+use PhpParser\Node\Scalar\InterpolatedString;
+use PhpParser\Node\Scalar\String_;
 use PhpParser\Node\Stmt\GroupUse;
 use PhpParser\Node\Stmt\Use_;
 use PhpParser\Node\UseItem;
@@ -10,7 +15,20 @@ use PhpParser\Node\UseItem;
 class NodeDumper {
     private bool $dumpComments;
     private bool $dumpPositions;
+    private bool $dumpOtherAttributes;
     private ?string $code;
+    private string $res;
+    private string $nl;
+
+    private const IGNORE_ATTRIBUTES = [
+        'comments' => true,
+        'startLine' => true,
+        'endLine' => true,
+        'startFilePos' => true,
+        'endFilePos' => true,
+        'startTokenPos' => true,
+        'endTokenPos' => true,
+    ];
 
     /**
      * Constructs a NodeDumper.
@@ -19,12 +37,14 @@ class NodeDumper {
      *  * bool dumpComments: Whether comments should be dumped.
      *  * bool dumpPositions: Whether line/offset information should be dumped. To dump offset
      *                        information, the code needs to be passed to dump().
+     *  * bool dumpOtherAttributes: Whether non-comment, non-position attributes should be dumped.
      *
      * @param array $options Options (see description)
      */
     public function __construct(array $options = []) {
         $this->dumpComments = !empty($options['dumpComments']);
         $this->dumpPositions = !empty($options['dumpPositions']);
+        $this->dumpOtherAttributes = !empty($options['dumpOtherAttributes']);
     }
 
     /**
@@ -39,72 +59,107 @@ class NodeDumper {
      */
     public function dump($node, ?string $code = null): string {
         $this->code = $code;
-        return $this->dumpRecursive($node);
+        $this->res = '';
+        $this->nl = "\n";
+        $this->dumpRecursive($node, false);
+        return $this->res;
     }
 
-    /** @param Node|Comment|array $node */
-    protected function dumpRecursive($node): string {
+    /** @param mixed $node */
+    protected function dumpRecursive($node, bool $indent = true): void {
+        if ($indent) {
+            $this->nl .= "    ";
+        }
         if ($node instanceof Node) {
-            $r = $node->getType();
+            $this->res .= $node->getType();
             if ($this->dumpPositions && null !== $p = $this->dumpPosition($node)) {
-                $r .= $p;
+                $this->res .= $p;
             }
-            $r .= '(';
+            $this->res .= '(';
 
             foreach ($node->getSubNodeNames() as $key) {
-                $r .= "\n    " . $key . ': ';
+                $this->res .= "$this->nl    " . $key . ': ';
 
                 $value = $node->$key;
-                if (null === $value) {
-                    $r .= 'null';
-                } elseif (false === $value) {
-                    $r .= 'false';
-                } elseif (true === $value) {
-                    $r .= 'true';
-                } elseif (is_scalar($value)) {
+                if (\is_int($value)) {
                     if ('flags' === $key || 'newModifier' === $key) {
-                        $r .= $this->dumpFlags($value);
-                    } elseif ('type' === $key && $node instanceof Include_) {
-                        $r .= $this->dumpIncludeType($value);
-                    } elseif ('type' === $key
-                            && ($node instanceof Use_ || $node instanceof UseItem || $node instanceof GroupUse)) {
-                        $r .= $this->dumpUseType($value);
-                    } else {
-                        $r .= $value;
+                        $this->res .= $this->dumpFlags($value);
+                        continue;
+                    }
+                    if ('type' === $key && $node instanceof Include_) {
+                        $this->res .= $this->dumpIncludeType($value);
+                        continue;
+                    }
+                    if ('type' === $key
+                            && ($node instanceof Use_ || $node instanceof UseItem || $node instanceof GroupUse)) {
+                        $this->res .= $this->dumpUseType($value);
+                        continue;
                     }
-                } else {
-                    $r .= str_replace("\n", "\n    ", $this->dumpRecursive($value));
                 }
+                $this->dumpRecursive($value);
             }
 
             if ($this->dumpComments && $comments = $node->getComments()) {
-                $r .= "\n    comments: " . str_replace("\n", "\n    ", $this->dumpRecursive($comments));
+                $this->res .= "$this->nl    comments: ";
+                $this->dumpRecursive($comments);
             }
-        } elseif (is_array($node)) {
-            $r = 'array(';
 
-            foreach ($node as $key => $value) {
-                $r .= "\n    " . $key . ': ';
+            if ($this->dumpOtherAttributes) {
+                foreach ($node->getAttributes() as $key => $value) {
+                    if (isset(self::IGNORE_ATTRIBUTES[$key])) {
+                        continue;
+                    }
 
-                if (null === $value) {
-                    $r .= 'null';
-                } elseif (false === $value) {
-                    $r .= 'false';
-                } elseif (true === $value) {
-                    $r .= 'true';
-                } elseif (is_scalar($value)) {
-                    $r .= $value;
-                } else {
-                    $r .= str_replace("\n", "\n    ", $this->dumpRecursive($value));
+                    $this->res .= "$this->nl    $key: ";
+                    if (\is_int($value)) {
+                        if ('kind' === $key) {
+                            if ($node instanceof Int_) {
+                                $this->res .= $this->dumpIntKind($value);
+                                continue;
+                            }
+                            if ($node instanceof String_ || $node instanceof InterpolatedString) {
+                                $this->res .= $this->dumpStringKind($value);
+                                continue;
+                            }
+                            if ($node instanceof Array_) {
+                                $this->res .= $this->dumpArrayKind($value);
+                                continue;
+                            }
+                            if ($node instanceof List_) {
+                                $this->res .= $this->dumpListKind($value);
+                                continue;
+                            }
+                        }
+                    }
+                    $this->dumpRecursive($value);
                 }
             }
+            $this->res .= "$this->nl)";
+        } elseif (\is_array($node)) {
+            $this->res .= 'array(';
+            foreach ($node as $key => $value) {
+                $this->res .= "$this->nl    " . $key . ': ';
+                $this->dumpRecursive($value);
+            }
+            $this->res .= "$this->nl)";
         } elseif ($node instanceof Comment) {
-            return $node->getReformattedText();
+            $this->res .= \str_replace("\n", $this->nl, $node->getReformattedText());
+        } elseif (\is_string($node)) {
+            $this->res .= \str_replace("\n", $this->nl, (string)$node);
+        } elseif (\is_int($node) || \is_float($node)) {
+            $this->res .= $node;
+        } elseif (null === $node) {
+            $this->res .= 'null';
+        } elseif (false === $node) {
+            $this->res .= 'false';
+        } elseif (true === $node) {
+            $this->res .= 'true';
         } else {
             throw new \InvalidArgumentException('Can only dump nodes and arrays.');
         }
-
-        return $r . "\n)";
+        if ($indent) {
+            $this->nl = \substr($this->nl, 0, -4);
+        }
     }
 
     protected function dumpFlags(int $flags): string {
@@ -138,32 +193,62 @@ class NodeDumper {
         }
     }
 
-    protected function dumpIncludeType(int $type): string {
-        $map = [
+    /** @param array<int, string> $map */
+    private function dumpEnum(int $value, array $map): string {
+        if (!isset($map[$value])) {
+            return (string) $value;
+        }
+        return $map[$value] . ' (' . $value . ')';
+    }
+
+    private function dumpIncludeType(int $type): string {
+        return $this->dumpEnum($type, [
             Include_::TYPE_INCLUDE      => 'TYPE_INCLUDE',
             Include_::TYPE_INCLUDE_ONCE => 'TYPE_INCLUDE_ONCE',
             Include_::TYPE_REQUIRE      => 'TYPE_REQUIRE',
             Include_::TYPE_REQUIRE_ONCE => 'TYPE_REQUIRE_ONCE',
-        ];
-
-        if (!isset($map[$type])) {
-            return (string) $type;
-        }
-        return $map[$type] . ' (' . $type . ')';
+        ]);
     }
 
-    protected function dumpUseType(int $type): string {
-        $map = [
+    private function dumpUseType(int $type): string {
+        return $this->dumpEnum($type, [
             Use_::TYPE_UNKNOWN  => 'TYPE_UNKNOWN',
             Use_::TYPE_NORMAL   => 'TYPE_NORMAL',
             Use_::TYPE_FUNCTION => 'TYPE_FUNCTION',
             Use_::TYPE_CONSTANT => 'TYPE_CONSTANT',
-        ];
+        ]);
+    }
 
-        if (!isset($map[$type])) {
-            return (string) $type;
-        }
-        return $map[$type] . ' (' . $type . ')';
+    private function dumpIntKind(int $kind): string {
+        return $this->dumpEnum($kind, [
+            Int_::KIND_BIN => 'KIND_BIN',
+            Int_::KIND_OCT => 'KIND_OCT',
+            Int_::KIND_DEC => 'KIND_DEC',
+            Int_::KIND_HEX => 'KIND_HEX',
+        ]);
+    }
+
+    private function dumpStringKind(int $kind): string {
+        return $this->dumpEnum($kind, [
+            String_::KIND_SINGLE_QUOTED => 'KIND_SINGLE_QUOTED',
+            String_::KIND_DOUBLE_QUOTED => 'KIND_DOUBLE_QUOTED',
+            String_::KIND_HEREDOC => 'KIND_HEREDOC',
+            String_::KIND_NOWDOC => 'KIND_NOWDOC',
+        ]);
+    }
+
+    private function dumpArrayKind(int $kind): string {
+        return $this->dumpEnum($kind, [
+            Array_::KIND_LONG => 'KIND_LONG',
+            Array_::KIND_SHORT => 'KIND_SHORT',
+        ]);
+    }
+
+    private function dumpListKind(int $kind): string {
+        return $this->dumpEnum($kind, [
+            List_::KIND_LIST => 'KIND_LIST',
+            List_::KIND_ARRAY => 'KIND_ARRAY',
+        ]);
     }
 
     /**

lib/PhpParser/NodeFinder.php

@@ -15,6 +15,10 @@ class NodeFinder {
      * @return Node[] Found nodes satisfying the filter callback
      */
     public function find($nodes, callable $filter): array {
+        if ($nodes === []) {
+            return [];
+        }
+
         if (!is_array($nodes)) {
             $nodes = [$nodes];
         }
@@ -52,6 +56,10 @@ class NodeFinder {
      * @return null|Node Found node (or null if none found)
      */
     public function findFirst($nodes, callable $filter): ?Node {
+        if ($nodes === []) {
+            return null;
+        }
+
         if (!is_array($nodes)) {
             $nodes = [$nodes];
         }

lib/PhpParser/NodeTraverser.php

@@ -34,7 +34,7 @@ class NodeTraverser implements NodeTraverserInterface {
      *
      * @param NodeVisitor ...$visitors Node visitors
      */
-    public function __construct(NodeVisitor... $visitors) {
+    public function __construct(NodeVisitor ...$visitors) {
         $this->visitors = $visitors;
     }
 
@@ -175,7 +175,7 @@ class NodeTraverser implements NodeTraverserInterface {
     protected function traverseArray(array $nodes): array {
         $doNodes = [];
 
-        foreach ($nodes as $i => &$node) {
+        foreach ($nodes as $i => $node) {
             if ($node instanceof Node) {
                 $traverseChildren = true;
                 $visitorIndex = -1;
@@ -185,7 +185,7 @@ class NodeTraverser implements NodeTraverserInterface {
                     if (null !== $return) {
                         if ($return instanceof Node) {
                             $this->ensureReplacementReasonable($node, $return);
-                            $node = $return;
+                            $nodes[$i] = $node = $return;
                         } elseif (\is_array($return)) {
                             $doNodes[] = [$i, $return];
                             continue 2;
@@ -225,7 +225,7 @@ class NodeTraverser implements NodeTraverserInterface {
                     if (null !== $return) {
                         if ($return instanceof Node) {
                             $this->ensureReplacementReasonable($node, $return);
-                            $node = $return;
+                            $nodes[$i] = $node = $return;
                         } elseif (\is_array($return)) {
                             $doNodes[] = [$i, $return];
                             break;

lib/PhpParser/NodeVisitor/CommentAnnotatingVisitor.php

@@ -0,0 +1,82 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\NodeVisitor;
+
+use PhpParser\Comment;
+use PhpParser\Node;
+use PhpParser\NodeVisitorAbstract;
+use PhpParser\Token;
+
+class CommentAnnotatingVisitor extends NodeVisitorAbstract {
+    /** @var int Last seen token start position */
+    private int $pos = 0;
+    /** @var Token[] Token array */
+    private array $tokens;
+    /** @var list<int> Token positions of comments */
+    private array $commentPositions = [];
+
+    /**
+     * Create a comment annotation visitor.
+     *
+     * @param Token[] $tokens Token array
+     */
+    public function __construct(array $tokens) {
+        $this->tokens = $tokens;
+
+        // Collect positions of comments. We use this to avoid traversing parts of the AST where
+        // there are no comments.
+        foreach ($tokens as $i => $token) {
+            if ($token->id === \T_COMMENT || $token->id === \T_DOC_COMMENT) {
+                $this->commentPositions[] = $i;
+            }
+        }
+    }
+
+    public function enterNode(Node $node) {
+        $nextCommentPos = current($this->commentPositions);
+        if ($nextCommentPos === false) {
+            // No more comments.
+            return self::STOP_TRAVERSAL;
+        }
+
+        $oldPos = $this->pos;
+        $this->pos = $pos = $node->getStartTokenPos();
+        if ($nextCommentPos > $oldPos && $nextCommentPos < $pos) {
+            $comments = [];
+            while (--$pos >= $oldPos) {
+                $token = $this->tokens[$pos];
+                if ($token->id === \T_DOC_COMMENT) {
+                    $comments[] = new Comment\Doc(
+                        $token->text, $token->line, $token->pos, $pos,
+                        $token->getEndLine(), $token->getEndPos() - 1, $pos);
+                    continue;
+                }
+                if ($token->id === \T_COMMENT) {
+                    $comments[] = new Comment(
+                        $token->text, $token->line, $token->pos, $pos,
+                        $token->getEndLine(), $token->getEndPos() - 1, $pos);
+                    continue;
+                }
+                if ($token->id !== \T_WHITESPACE) {
+                    break;
+                }
+            }
+            if (!empty($comments)) {
+                $node->setAttribute('comments', array_reverse($comments));
+            }
+
+            do {
+                $nextCommentPos = next($this->commentPositions);
+            } while ($nextCommentPos !== false && $nextCommentPos < $this->pos);
+        }
+
+        $endPos = $node->getEndTokenPos();
+        if ($nextCommentPos > $endPos) {
+            // Skip children if there are no comments located inside this node.
+            $this->pos = $endPos;
+            return self::DONT_TRAVERSE_CHILDREN;
+        }
+
+        return null;
+    }
+}

lib/PhpParser/NodeVisitor/NameResolver.php

@@ -115,6 +115,9 @@ class NameResolver extends NodeVisitorAbstract {
                 $this->addNamespacedName($const);
             }
         } elseif ($node instanceof Stmt\ClassConst) {
+            if (null !== $node->type) {
+                $node->type = $this->resolveType($node->type);
+            }
             $this->resolveAttrGroups($node);
         } elseif ($node instanceof Stmt\EnumCase) {
             $this->resolveAttrGroups($node);

lib/PhpParser/ParserAbstract.php

@@ -30,6 +30,7 @@ use PhpParser\Node\Stmt\Nop;
 use PhpParser\Node\Stmt\Property;
 use PhpParser\Node\Stmt\TryCatch;
 use PhpParser\Node\UseItem;
+use PhpParser\NodeVisitor\CommentAnnotatingVisitor;
 
 abstract class ParserAbstract implements Parser {
     private const SYMBOL_NONE = -1;
@@ -201,6 +202,11 @@ abstract class ParserAbstract implements Parser {
         $this->semValue = null;
         $this->createdArrays = null;
 
+        if ($result !== null) {
+            $traverser = new NodeTraverser(new CommentAnnotatingVisitor($this->tokens));
+            $traverser->traverse($result);
+        }
+
         return $result;
     }
 
@@ -312,7 +318,7 @@ abstract class ParserAbstract implements Parser {
                     try {
                         $callback = $this->reduceCallbacks[$rule];
                         if ($callback !== null) {
-                            $callback($stackPos);
+                            $callback($this, $stackPos);
                         } elseif ($ruleLength > 0) {
                             $this->semValue = $this->semStack[$stackPos - $ruleLength + 1];
                         }
@@ -473,7 +479,7 @@ abstract class ParserAbstract implements Parser {
     protected function getAttributes(int $tokenStartPos, int $tokenEndPos): array {
         $startToken = $this->tokens[$tokenStartPos];
         $afterEndToken = $this->tokens[$tokenEndPos + 1];
-        $attributes = [
+        return [
             'startLine' => $startToken->line,
             'startTokenPos' => $tokenStartPos,
             'startFilePos' => $startToken->pos,
@@ -481,11 +487,6 @@ abstract class ParserAbstract implements Parser {
             'endTokenPos' => $tokenEndPos,
             'endFilePos' => $afterEndToken->pos - 1,
         ];
-        $comments = $this->getCommentsBeforeToken($tokenStartPos);
-        if (!empty($comments)) {
-            $attributes['comments'] = $comments;
-        }
-        return $attributes;
     }
 
     /**
@@ -500,7 +501,7 @@ abstract class ParserAbstract implements Parser {
 
         // Get attributes for the sentinel token.
         $token = $this->tokens[$tokenPos];
-        $attributes = [
+        return [
             'startLine' => $token->line,
             'startTokenPos' => $tokenPos,
             'startFilePos' => $token->pos,
@@ -508,11 +509,6 @@ abstract class ParserAbstract implements Parser {
             'endTokenPos' => $tokenPos,
             'endFilePos' => $token->pos,
         ];
-        $comments = $this->getCommentsBeforeToken($tokenPos);
-        if (!empty($comments)) {
-            $attributes['comments'] = $comments;
-        }
-        return $attributes;
     }
 
     /*
@@ -585,7 +581,7 @@ abstract class ParserAbstract implements Parser {
         } else {
             // For semicolon namespaces we have to move the statements after a namespace declaration into ->stmts
             $resultStmts = [];
-            $targetStmts =& $resultStmts;
+            $targetStmts = &$resultStmts;
             $lastNs = null;
             foreach ($stmts as $stmt) {
                 if ($stmt instanceof Node\Stmt\Namespace_) {
@@ -594,12 +590,12 @@ abstract class ParserAbstract implements Parser {
                     }
                     if ($stmt->stmts === null) {
                         $stmt->stmts = [];
-                        $targetStmts =& $stmt->stmts;
+                        $targetStmts = &$stmt->stmts;
                         $resultStmts[] = $stmt;
                     } else {
                         // This handles the invalid case of mixed style namespaces
                         $resultStmts[] = $stmt;
-                        $targetStmts =& $resultStmts;
+                        $targetStmts = &$resultStmts;
                     }
                     $lastNs = $stmt;
                 } elseif ($stmt instanceof Node\Stmt\HaltCompiler) {
@@ -845,6 +841,7 @@ abstract class ParserAbstract implements Parser {
 
         if (\is_string($contents)) {
             if ($contents === '') {
+                $attributes['rawValue'] = $contents;
                 return new String_('', $attributes);
             }
 
@@ -852,6 +849,7 @@ abstract class ParserAbstract implements Parser {
                 $contents, $indentLen, $indentChar, true, true, $attributes
             );
             $contents = preg_replace('~(\r\n|\n|\r)\z~', '', $contents);
+            $attributes['rawValue'] = $contents;
 
             if ($kind === String_::KIND_HEREDOC) {
                 $contents = String_::parseEscapeSequences($contents, null, $parseUnicodeEscape);
@@ -878,6 +876,7 @@ abstract class ParserAbstract implements Parser {
                     if ($isLast) {
                         $part->value = preg_replace('~(\r\n|\n|\r)\z~', '', $part->value);
                     }
+                    $part->setAttribute('rawValue', $part->value);
                     $part->value = String_::parseEscapeSequences($part->value, null, $parseUnicodeEscape);
                     if ('' === $part->value) {
                         continue;
@@ -899,12 +898,9 @@ abstract class ParserAbstract implements Parser {
     }
 
     /**
-     * Get comments before the given token position.
-     *
-     * @return Comment[] Comments
+     * Get last comment before the given token position, if any
      */
-    protected function getCommentsBeforeToken(int $tokenPos): array {
-        $comments = [];
+    protected function getCommentBeforeToken(int $tokenPos): ?Comment {
         while (--$tokenPos >= 0) {
             $token = $this->tokens[$tokenPos];
             if (!isset($this->dropTokens[$token->id])) {
@@ -912,22 +908,21 @@ abstract class ParserAbstract implements Parser {
             }
 
             if ($token->id === \T_COMMENT || $token->id === \T_DOC_COMMENT) {
-                $comments[] = $this->createCommentFromToken($token, $tokenPos);
+                return $this->createCommentFromToken($token, $tokenPos);
             }
         }
-        return \array_reverse($comments);
+        return null;
     }
 
     /**
      * Create a zero-length nop to capture preceding comments, if any.
      */
     protected function maybeCreateZeroLengthNop(int $tokenPos): ?Nop {
-        $comments = $this->getCommentsBeforeToken($tokenPos);
-        if (empty($comments)) {
+        $comment = $this->getCommentBeforeToken($tokenPos);
+        if ($comment === null) {
             return null;
         }
 
-        $comment = $comments[\count($comments) - 1];
         $commentEndLine = $comment->getEndLine();
         $commentEndFilePos = $comment->getEndFilePos();
         $commentEndTokenPos = $comment->getEndTokenPos();
@@ -938,14 +933,12 @@ abstract class ParserAbstract implements Parser {
             'endFilePos' => $commentEndFilePos,
             'startTokenPos' => $commentEndTokenPos + 1,
             'endTokenPos' => $commentEndTokenPos,
-            'comments' => $comments,
         ];
         return new Nop($attributes);
     }
 
     protected function maybeCreateNop(int $tokenStartPos, int $tokenEndPos): ?Nop {
-        $comments = $this->getCommentsBeforeToken($tokenStartPos);
-        if (empty($comments)) {
+        if ($this->getCommentBeforeToken($tokenStartPos) === null) {
             return null;
         }
         return new Nop($this->getAttributes($tokenStartPos, $tokenEndPos));

lib/PhpParser/ParserFactory.php

@@ -6,34 +6,6 @@ use PhpParser\Parser\Php7;
 use PhpParser\Parser\Php8;
 
 class ParserFactory {
-    public const PREFER_PHP7 = 1;
-    public const ONLY_PHP7 = 3;
-
-    /**
-     * Creates a Parser instance, according to the provided kind.
-     *
-     * @param int $kind One of ::PREFER_PHP7 or ::ONLY_PHP7
-     * @param Lexer|null $lexer Lexer to use. Defaults to emulative lexer when not specified
-     *
-     * @return Parser The parser instance
-     *
-     * @deprecated Use createForVersion(), createForNewestSupportedVersion() or createForHostVersion() instead.
-     */
-    public function create(int $kind, ?Lexer $lexer = null): Parser {
-        if (null === $lexer) {
-            $lexer = new Lexer\Emulative();
-        }
-        switch ($kind) {
-            case self::PREFER_PHP7:
-            case self::ONLY_PHP7:
-                return new Parser\Php7($lexer);
-            default:
-                throw new \LogicException(
-                    'Kind must be one of ::PREFER_PHP7 or ::ONLY_PHP7'
-                );
-        }
-    }
-
     /**
      * Create a parser targeting the given version on a best-effort basis. The parser will generally
      * accept code for the newest supported version, but will try to accommodate code that becomes

lib/PhpParser/PhpVersion.php

@@ -43,7 +43,7 @@ class PhpVersion {
      * if it is still under development.
      */
     public static function getNewestSupported(): self {
-        return self::fromComponents(8, 2);
+        return self::fromComponents(8, 3);
     }
 
     /**
@@ -149,9 +149,16 @@ class PhpVersion {
     }
 
     /**
-     * Whether this version support yield in expression context without parentheses.
+     * Whether this version supports yield in expression context without parentheses.
      */
     public function supportsYieldWithoutParentheses(): bool {
         return $this->id >= 70000;
     }
+
+    /**
+     * Whether this version supports unicode escape sequences in strings.
+     */
+    public function supportsUnicodeEscapes(): bool {
+        return $this->id >= 70000;
+    }
 }

lib/PhpParser/PrettyPrinter/Standard.php

@@ -960,10 +960,6 @@ class Standard extends PrettyPrinterAbstract {
         return 'return' . (null !== $node->expr ? ' ' . $this->p($node->expr) : '') . ';';
     }
 
-    protected function pStmt_Throw(Stmt\Throw_ $node): string {
-        return 'throw ' . $this->p($node->expr) . ';';
-    }
-
     protected function pStmt_Label(Stmt\Label $node): string {
         return $node->name . ':';
     }
@@ -1012,6 +1008,10 @@ class Standard extends PrettyPrinterAbstract {
         return '';
     }
 
+    protected function pStmt_Block(Stmt\Block $node): string {
+        return '{' . $this->pStmts($node->stmts) . $this->nl . '}';
+    }
+
     // Helpers
 
     protected function pClassCommon(Stmt\Class_ $node, string $afterClassToken): string {

lib/PhpParser/PrettyPrinterAbstract.php

@@ -826,9 +826,8 @@ abstract class PrettyPrinterAbstract implements PrettyPrinter {
                 }
 
                 if ($skipRemovedNode) {
-                    if ($isStmtList && ($this->origTokens->haveBracesInRange($pos, $itemStartPos) ||
-                                        $this->origTokens->haveTagInRange($pos, $itemStartPos))) {
-                        // We'd remove the brace of a code block.
+                    if ($isStmtList && $this->origTokens->haveTagInRange($pos, $itemStartPos)) {
+                        // We'd remove an opening/closing PHP tag.
                         // TODO: Preserve formatting.
                         $this->setIndentLevel($origIndentLevel);
                         return null;
@@ -937,9 +936,8 @@ abstract class PrettyPrinterAbstract implements PrettyPrinter {
                         $pos, $itemStartPos, $indentAdjustment);
                     $skipRemovedNode = true;
                 } else {
-                    if ($isStmtList && ($this->origTokens->haveBracesInRange($pos, $itemStartPos) ||
-                                        $this->origTokens->haveTagInRange($pos, $itemStartPos))) {
-                        // We'd remove the brace of a code block.
+                    if ($isStmtList && $this->origTokens->haveTagInRange($pos, $itemStartPos)) {
+                        // We'd remove an opening/closing PHP tag.
                         // TODO: Preserve formatting.
                         return null;
                     }
@@ -1540,6 +1538,7 @@ abstract class PrettyPrinterAbstract implements PrettyPrinter {
             Stmt\Function_::class . '->stmts' => "\n",
             Stmt\If_::class . '->stmts' => "\n",
             Stmt\Namespace_::class . '->stmts' => "\n",
+            Stmt\Block::class . '->stmts' => "\n",
 
             // Attribute groups
             Stmt\Class_::class . '->attrGroups' => "\n",

lib/PhpParser/compatibility_tokens.php

@@ -26,6 +26,13 @@ if (!\function_exists('PhpParser\defineCompatibilityTokens')) {
         foreach ($compatTokens as $token) {
             if (\defined($token)) {
                 $tokenId = \constant($token);
+                if (!\is_int($tokenId)) {
+                    throw new \Error(sprintf(
+                        'Token %s has ID of type %s, should be int. ' .
+                        'You may be using a library with broken token emulation',
+                        $token, \gettype($tokenId)
+                    ));
+                }
                 $clashingToken = $usedTokenIds[$tokenId] ?? null;
                 if ($clashingToken !== null) {
                     throw new \Error(sprintf(

phpstan-baseline.neon

@@ -100,11 +100,6 @@ parameters:
 			count: 1
 			path: lib/PhpParser/NodeDumper.php
 
-		-
-			message: "#^Method PhpParser\\\\NodeDumper\\:\\:dumpRecursive\\(\\) has parameter \\$node with no value type specified in iterable type array\\.$#"
-			count: 1
-			path: lib/PhpParser/NodeDumper.php
-
 		-
 			message: "#^Method PhpParser\\\\NodeTraverser\\:\\:traverseArray\\(\\) has parameter \\$nodes with no value type specified in iterable type array\\.$#"
 			count: 1

test/PhpParser/CodeParsingTest.php

@@ -28,6 +28,7 @@ class CodeParsingTest extends CodeTestAbstract {
     // Must be public for updateTests.php
     public function getParseOutput(Parser $parser, $code, array $modes) {
         $dumpPositions = isset($modes['positions']);
+        $dumpOtherAttributes = isset($modes['attributes']);
 
         $errors = new ErrorHandler\Collecting();
         $stmts = $parser->parse($code, $errors);
@@ -38,7 +39,11 @@ class CodeParsingTest extends CodeTestAbstract {
         }
 
         if (null !== $stmts) {
-            $dumper = new NodeDumper(['dumpComments' => true, 'dumpPositions' => $dumpPositions]);
+            $dumper = new NodeDumper([
+                'dumpComments' => true,
+                'dumpPositions' => $dumpPositions,
+                'dumpOtherAttributes' => $dumpOtherAttributes,
+            ]);
             $output .= $dumper->dump($stmts, $code);
         }
 

test/PhpParser/Node/Scalar/DNumberTest.php

@@ -8,7 +8,7 @@ use PhpParser\ParserFactory;
 
 class DNumberTest extends \PHPUnit\Framework\TestCase {
     public function testRawValue() {
-        $parser = (new ParserFactory())->create(ParserFactory::PREFER_PHP7);
+        $parser = (new ParserFactory())->createForNewestSupportedVersion();
         $nodes = $parser->parse('<?php echo 1_234.56;');
 
         $echo = $nodes[0];

test/PhpParser/Node/Scalar/NumberTest.php

@@ -7,7 +7,7 @@ use PhpParser\ParserFactory;
 
 class NumberTest extends \PHPUnit\Framework\TestCase {
     public function testRawValue() {
-        $parser = (new ParserFactory())->create(ParserFactory::PREFER_PHP7);
+        $parser = (new ParserFactory())->createForNewestSupportedVersion();
         $nodes = $parser->parse('<?php echo 1_234;');
 
         $echo = $nodes[0];

test/PhpParser/Node/Scalar/StringTest.php

@@ -7,7 +7,7 @@ use PhpParser\ParserFactory;
 
 class StringTest extends \PHPUnit\Framework\TestCase {
     public function testRawValue() {
-        $parser = (new ParserFactory())->create(ParserFactory::PREFER_PHP7);
+        $parser = (new ParserFactory())->createForNewestSupportedVersion();
         $nodes = $parser->parse('<?php echo "sequence \x41";');
 
         $echo = $nodes[0];

test/PhpParser/NodeAbstractTest.php

@@ -63,9 +63,9 @@ class NodeAbstractTest extends \PHPUnit\Framework\TestCase {
         $this->assertSame('/** doc comment */', $node->getDocComment()->getText());
         $this->assertSame('value1', $node->subNode1);
         $this->assertSame('value2', $node->subNode2);
-        $this->assertObjectHasAttribute('subNode1', $node);
-        $this->assertObjectHasAttribute('subNode2', $node);
-        $this->assertObjectNotHasAttribute('subNode3', $node);
+        $this->assertTrue(isset($node->subNode1));
+        $this->assertTrue(isset($node->subNode2));
+        $this->assertTrue(!isset($node->subNode3));
         $this->assertSame($attributes, $node->getAttributes());
         $this->assertSame($attributes['comments'], $node->getComments());
 
@@ -125,7 +125,7 @@ class NodeAbstractTest extends \PHPUnit\Framework\TestCase {
         $this->assertSame('newValue', $node->subNode1);
 
         // indirect modification
-        $subNode =& $node->subNode1;
+        $subNode = &$node->subNode1;
         $subNode = 'newNewValue';
         $this->assertSame('newNewValue', $node->subNode1);
 

test/PhpParser/NodeVisitor/NameResolverTest.php

@@ -188,7 +188,7 @@ class A extends B implements C, D {
         E::h as i;
         E::j insteadof F, G;
     }
-    
+
     #[X]
     public float $php = 7.4;
     public ?Foo $person;
@@ -197,6 +197,10 @@ class A extends B implements C, D {
 
     #[X]
     const C = 1;
+    
+    public const X A = X::Bar;
+    public const X\Foo B = X\Foo::Bar;
+    public const \X\Foo C = \X\Foo::Bar;
 }
 
 #[X]
@@ -262,6 +266,9 @@ class A extends \NS\B implements \NS\C, \NS\D
     public \NS\A|\NS\B|int $prop;
     #[\NS\X]
     const C = 1;
+    public const \NS\X A = \NS\X::Bar;
+    public const \NS\X\Foo B = \NS\X\Foo::Bar;
+    public const \X\Foo C = \X\Foo::Bar;
 }
 #[\NS\X]
 interface A extends \NS\C, \NS\D

test/PhpParser/NodeVisitor/NodeConnectingVisitorTest.php

@@ -11,7 +11,7 @@ use PhpParser\ParserFactory;
 
 final class NodeConnectingVisitorTest extends \PHPUnit\Framework\TestCase {
     public function testConnectsNodeToItsParentNodeAndItsSiblingNodes() {
-        $ast = (new ParserFactory())->create(ParserFactory::PREFER_PHP7)->parse(
+        $ast = (new ParserFactory())->createForNewestSupportedVersion()->parse(
             '<?php if (true) {} else {}'
         );
 

test/PhpParser/NodeVisitor/ParentConnectingVisitorTest.php

@@ -9,7 +9,7 @@ use PhpParser\ParserFactory;
 
 final class ParentConnectingVisitorTest extends \PHPUnit\Framework\TestCase {
     public function testConnectsChildNodeToParentNode() {
-        $ast = (new ParserFactory())->create(ParserFactory::PREFER_PHP7)->parse(
+        $ast = (new ParserFactory())->createForNewestSupportedVersion()->parse(
             '<?php class C { public function m() {} }'
         );
 

test/PhpParser/ParserFactoryTest.php

@@ -5,23 +5,13 @@ namespace PhpParser;
 /* This test is very weak, because PHPUnit's assertEquals assertion is way too slow dealing with the
  * large objects involved here. So we just do some basic instanceof tests instead. */
 
-class ParserFactoryTest extends \PHPUnit\Framework\TestCase {
-    /** @dataProvider provideTestCreate */
-    public function testCreate($kind, $lexer, $expected) {
-        $this->assertInstanceOf($expected, (new ParserFactory())->create($kind, $lexer));
-    }
+use PhpParser\Parser\Php7;
+use PhpParser\Parser\Php8;
 
-    public function provideTestCreate() {
-        $lexer = new Lexer();
-        return [
-            [
-                ParserFactory::PREFER_PHP7, $lexer,
-                Parser\Php7::class
-            ],
-            [
-                ParserFactory::ONLY_PHP7, null,
-                Parser\Php7::class
-            ],
-        ];
+class ParserFactoryTest extends \PHPUnit\Framework\TestCase {
+    public function testCreate() {
+        $factory = new ParserFactory();
+        $this->assertInstanceOf(Php8::class, $factory->createForNewestSupportedVersion());
+        $this->assertInstanceOf(Parser::class, $factory->createForHostVersion());
     }
 }

test/PhpParser/PrettyPrinterTest.php

@@ -28,7 +28,6 @@ class PrettyPrinterTest extends CodeTestAbstract {
 
     /**
      * @dataProvider provideTestPrettyPrint
-     * @covers \PhpParser\PrettyPrinter\Standard<extended>
      */
     public function testPrettyPrint($name, $code, $expected, $mode) {
         $this->doTestPrettyPrintMethod('prettyPrint', $name, $code, $expected, $mode);
@@ -36,7 +35,6 @@ class PrettyPrinterTest extends CodeTestAbstract {
 
     /**
      * @dataProvider provideTestPrettyPrintFile
-     * @covers \PhpParser\PrettyPrinter\Standard<extended>
      */
     public function testPrettyPrintFile($name, $code, $expected, $mode) {
         $this->doTestPrettyPrintMethod('prettyPrintFile', $name, $code, $expected, $mode);
@@ -181,7 +179,6 @@ class PrettyPrinterTest extends CodeTestAbstract {
 
     /**
      * @dataProvider provideTestFormatPreservingPrint
-     * @covers \PhpParser\PrettyPrinter\Standard<extended>
      */
     public function testFormatPreservingPrint($name, $code, $modification, $expected, $modeLine) {
         $lexer = new Lexer\Emulative();
@@ -218,7 +215,6 @@ CODE
 
     /**
      * @dataProvider provideTestRoundTripPrint
-     * @covers \PhpParser\PrettyPrinter\Standard<extended>
      */
     public function testRoundTripPrint($name, $code, $expected, $modeLine) {
         /**

test/code/formatPreservation/basic.test

@@ -153,8 +153,9 @@ $stmts[1] = $tmp;
  * fallback. */
 -----
 <?php
-
-$x;
+{
+    $x;
+}
 function foo() {
     foo();
     /*
@@ -186,4 +187,4 @@ echo "{$bar}bar";
 [$a
 ,$b
 ,
-,] = $b;
+,] = $b;

test/code/formatPreservation/comments.test

@@ -50,3 +50,14 @@ class Test {
         // some code
     }
 }
+-----
+<?php
+    class Example {
+    }
+-----
+$stmts[0]->setDocComment(new Comment\Doc("/** foo */"));
+-----
+<?php
+    /** foo */
+    class Example {
+    }

test/code/formatPreservation/listInsertion.test

@@ -354,3 +354,16 @@ array_splice($stmts[0]->expr->var->items, 1, 0, [null]);
 -----
 <?php
 list($x, , $y) = $z;
+-----
+<?php
+{
+    $a; $b;
+}
+-----
+$stmts[0]->stmts[] = new Stmt\Expression(new Expr\Variable('c'));
+-----
+<?php
+{
+    $a; $b;
+    $c;
+}

test/code/formatPreservation/listRemoval.test

@@ -70,7 +70,6 @@ $y;
 array_splice($stmts, 0, 1, []);
 -----
 <?php
-
 $y;
 -----
 <?php
@@ -80,8 +79,7 @@ $x;
 array_splice($stmts, 0, 1, []);
 -----
 <?php
-
-$y;
+{ $y; }
 -----
 <?php
 $x;
@@ -90,7 +88,6 @@ $x;
 array_pop($stmts);
 -----
 <?php
-
 $x;
 -----
 <?php
@@ -100,8 +97,7 @@ $y;
 array_pop($stmts);
 -----
 <?php
-
-$x;
+{ $x; }
 -----
 <?php
 // Foo

test/code/parser/blockComments.test

@@ -15,20 +15,31 @@ Comments on blocks
 {}
 -----
 array(
-    0: Stmt_Expression(
-        expr: Expr_Variable(
-            name: a
-            comments: array(
-                0: // baz
+    0: Stmt_Block(
+        stmts: array(
+            0: Stmt_Block(
+                stmts: array(
+                    0: Stmt_Expression(
+                        expr: Expr_Variable(
+                            name: a
+                        )
+                        comments: array(
+                            0: // baz
+                        )
+                    )
+                )
+                comments: array(
+                    0: // bar
+                )
             )
         )
         comments: array(
             0: // foo
-            1: // bar
-            2: // baz
         )
     )
-    1: Stmt_Nop(
+    1: Stmt_Block(
+        stmts: array(
+        )
         comments: array(
             0: // empty
         )

test/code/parser/comments.test

@@ -24,12 +24,6 @@ array(
     0: Stmt_Expression(
         expr: Expr_Variable(
             name: var
-            comments: array(
-                0: /** doc 1 */
-                1: /* foobar 1 */
-                2: // foo 1
-                3: // bar 1
-            )
         )
         comments: array(
             0: /** doc 1 */

test/code/parser/emptyFile.test

@@ -0,0 +1,6 @@
+Empty file
+-----
+
+-----
+array(
+)

test/code/parser/errorHandling/recovery.test

@@ -223,13 +223,17 @@ array(
             )
         )
     )
-    1: Stmt_Expression(
-        expr: Expr_Assign(
-            var: Expr_Variable(
-                name: j
-            )
-            expr: Scalar_Int(
-                value: 1
+    1: Stmt_Block(
+        stmts: array(
+            0: Stmt_Expression(
+                expr: Expr_Assign(
+                    var: Expr_Variable(
+                        name: j
+                    )
+                    expr: Scalar_Int(
+                        value: 1
+                    )
+                )
             )
         )
     )
@@ -521,9 +525,11 @@ array(
                     )
                 )
             )
-            12: Stmt_Throw(
-                expr: Expr_Variable(
-                    name: x
+            12: Stmt_Expression(
+                expr: Expr_Throw(
+                    expr: Expr_Variable(
+                        name: x
+                    )
                 )
             )
             13: Stmt_Goto(
@@ -832,11 +838,12 @@ array(
 <?php
 
 class Foo {
+    public $bar1;
     publi $foo;
     public $bar;
 }
 -----
-Syntax error, unexpected T_STRING from 4:5 to 4:9
+Syntax error, unexpected T_STRING from 5:5 to 5:9
 array(
     0: Stmt_Class(
         attrGroups: array(
@@ -850,6 +857,20 @@ array(
         )
         stmts: array(
             0: Stmt_Property(
+                attrGroups: array(
+                )
+                flags: PUBLIC (1)
+                type: null
+                props: array(
+                    0: PropertyItem(
+                        name: VarLikeIdentifier(
+                            name: bar1
+                        )
+                        default: null
+                    )
+                )
+            )
+            1: Stmt_Property(
                 attrGroups: array(
                 )
                 flags: PUBLIC (1)

test/code/parser/expr/arrayDef.test

@@ -116,9 +116,6 @@ array(
         expr: Expr_Array(
             items: array(
             )
-            comments: array(
-                0: // short array syntax
-            )
         )
         comments: array(
             0: // short array syntax

test/code/parser/expr/assign.test

@@ -41,16 +41,10 @@ array(
         expr: Expr_Assign(
             var: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // simple assign
-                )
             )
             expr: Expr_Variable(
                 name: b
             )
-            comments: array(
-                0: // simple assign
-            )
         )
         comments: array(
             0: // simple assign
@@ -60,16 +54,10 @@ array(
         expr: Expr_AssignOp_BitwiseAnd(
             var: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // combined assign
-                )
             )
             expr: Expr_Variable(
                 name: b
             )
-            comments: array(
-                0: // combined assign
-            )
         )
         comments: array(
             0: // combined assign
@@ -199,9 +187,6 @@ array(
         expr: Expr_Assign(
             var: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // chained assign
-                )
             )
             expr: Expr_AssignOp_Mul(
                 var: Expr_Variable(
@@ -216,9 +201,6 @@ array(
                     )
                 )
             )
-            comments: array(
-                0: // chained assign
-            )
         )
         comments: array(
             0: // chained assign
@@ -228,16 +210,10 @@ array(
         expr: Expr_AssignRef(
             var: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // by ref assign
-                )
             )
             expr: Expr_Variable(
                 name: b
             )
-            comments: array(
-                0: // by ref assign
-            )
         )
         comments: array(
             0: // by ref assign
@@ -256,16 +232,10 @@ array(
                         unpack: false
                     )
                 )
-                comments: array(
-                    0: // list() assign
-                )
             )
             expr: Expr_Variable(
                 name: b
             )
-            comments: array(
-                0: // list() assign
-            )
         )
         comments: array(
             0: // list() assign
@@ -349,9 +319,6 @@ array(
             var: Expr_Variable(
                 name: a
             )
-            comments: array(
-                0: // inc/dec
-            )
         )
         comments: array(
             0: // inc/dec

test/code/parser/expr/exprInIsset.test

@@ -17,9 +17,6 @@ array(
                     name: b
                 )
             )
-            comments: array(
-                0: // This is legal.
-            )
         )
         comments: array(
             0: // This is legal.
@@ -37,9 +34,6 @@ array(
                     )
                 )
             )
-            comments: array(
-                0: // This is illegal, but not a syntax error.
-            )
         )
         comments: array(
             0: // This is illegal, but not a syntax error.

test/code/parser/expr/exprInList.test

@@ -29,16 +29,10 @@ array(
                         unpack: false
                     )
                 )
-                comments: array(
-                    0: // This is legal.
-                )
             )
             expr: Expr_Variable(
                 name: x
             )
-            comments: array(
-                0: // This is legal.
-            )
         )
         comments: array(
             0: // This is legal.
@@ -62,16 +56,10 @@ array(
                         unpack: false
                     )
                 )
-                comments: array(
-                    0: // This is illegal, but not a syntax error.
-                )
             )
             expr: Expr_Variable(
                 name: x
             )
-            comments: array(
-                0: // This is illegal, but not a syntax error.
-            )
         )
         comments: array(
             0: // This is illegal, but not a syntax error.

test/code/parser/expr/fetchAndCall/funcCall.test

@@ -20,15 +20,9 @@ array(
         expr: Expr_FuncCall(
             name: Name(
                 name: a
-                comments: array(
-                    0: // function name variations
-                )
             )
             args: array(
             )
-            comments: array(
-                0: // function name variations
-            )
         )
         comments: array(
             0: // function name variations
@@ -130,22 +124,13 @@ array(
             var: Expr_FuncCall(
                 name: Name(
                     name: a
-                    comments: array(
-                        0: // array dereferencing
-                    )
                 )
                 args: array(
                 )
-                comments: array(
-                    0: // array dereferencing
-                )
             )
             dim: Scalar_String(
                 value: b
             )
-            comments: array(
-                0: // array dereferencing
-            )
         )
         comments: array(
             0: // array dereferencing

test/code/parser/expr/fetchAndCall/objectAccess.test

@@ -22,16 +22,10 @@ array(
         expr: Expr_PropertyFetch(
             var: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // property fetch variations
-                )
             )
             name: Identifier(
                 name: b
             )
-            comments: array(
-                0: // property fetch variations
-            )
         )
         comments: array(
             0: // property fetch variations
@@ -71,18 +65,12 @@ array(
         expr: Expr_MethodCall(
             var: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // method call variations
-                )
             )
             name: Identifier(
                 name: b
             )
             args: array(
             )
-            comments: array(
-                0: // method call variations
-            )
         )
         comments: array(
             0: // method call variations
@@ -136,25 +124,16 @@ array(
             var: Expr_MethodCall(
                 var: Expr_Variable(
                     name: a
-                    comments: array(
-                        0: // array dereferencing
-                    )
                 )
                 name: Identifier(
                     name: b
                 )
                 args: array(
                 )
-                comments: array(
-                    0: // array dereferencing
-                )
             )
             dim: Scalar_String(
                 value: c
             )
-            comments: array(
-                0: // array dereferencing
-            )
         )
         comments: array(
             0: // array dereferencing

test/code/parser/expr/fetchAndCall/staticCall.test

@@ -23,18 +23,12 @@ array(
         expr: Expr_StaticCall(
             class: Name(
                 name: A
-                comments: array(
-                    0: // method name variations
-                )
             )
             name: Identifier(
                 name: b
             )
             args: array(
             )
-            comments: array(
-                0: // method name variations
-            )
         )
         comments: array(
             0: // method name variations
@@ -112,25 +106,16 @@ array(
             var: Expr_StaticCall(
                 class: Name(
                     name: A
-                    comments: array(
-                        0: // array dereferencing
-                    )
                 )
                 name: Identifier(
                     name: b
                 )
                 args: array(
                 )
-                comments: array(
-                    0: // array dereferencing
-                )
             )
             dim: Scalar_String(
                 value: c
             )
-            comments: array(
-                0: // array dereferencing
-            )
         )
         comments: array(
             0: // array dereferencing
@@ -140,18 +125,12 @@ array(
         expr: Expr_StaticCall(
             class: Name(
                 name: static
-                comments: array(
-                    0: // class name variations
-                )
             )
             name: Identifier(
                 name: b
             )
             args: array(
             )
-            comments: array(
-                0: // class name variations
-            )
         )
         comments: array(
             0: // class name variations

test/code/parser/expr/fetchAndCall/staticPropertyFetch.test

@@ -18,16 +18,10 @@ array(
         expr: Expr_StaticPropertyFetch(
             class: Name(
                 name: A
-                comments: array(
-                    0: // property name variations
-                )
             )
             name: VarLikeIdentifier(
                 name: b
             )
-            comments: array(
-                0: // property name variations
-            )
         )
         comments: array(
             0: // property name variations
@@ -58,23 +52,14 @@ array(
             var: Expr_StaticPropertyFetch(
                 class: Name(
                     name: A
-                    comments: array(
-                        0: // array access
-                    )
                 )
                 name: VarLikeIdentifier(
                     name: b
                 )
-                comments: array(
-                    0: // array access
-                )
             )
             dim: Scalar_String(
                 value: c
             )
-            comments: array(
-                0: // array access
-            )
         )
         comments: array(
             0: // array access

test/code/parser/expr/firstClassCallables.test

@@ -61,9 +61,6 @@ array(
                 0: VariadicPlaceholder(
                 )
             )
-            comments: array(
-                0: // These are invalid, but accepted on the parser level.
-            )
         )
         comments: array(
             0: // These are invalid, but accepted on the parser level.

test/code/parser/expr/logic.test

@@ -25,16 +25,10 @@ array(
         expr: Expr_BinaryOp_BooleanAnd(
             left: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // boolean ops
-                )
             )
             right: Expr_Variable(
                 name: b
             )
-            comments: array(
-                0: // boolean ops
-            )
         )
         comments: array(
             0: // boolean ops
@@ -70,16 +64,10 @@ array(
         expr: Expr_BinaryOp_LogicalAnd(
             left: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // logical ops
-                )
             )
             right: Expr_Variable(
                 name: b
             )
-            comments: array(
-                0: // logical ops
-            )
         )
         comments: array(
             0: // logical ops
@@ -110,16 +98,10 @@ array(
             left: Expr_BinaryOp_BooleanAnd(
                 left: Expr_Variable(
                     name: a
-                    comments: array(
-                        0: // precedence
-                    )
                 )
                 right: Expr_Variable(
                     name: b
                 )
-                comments: array(
-                    0: // precedence
-                )
             )
             right: Expr_BinaryOp_BooleanAnd(
                 left: Expr_Variable(
@@ -129,9 +111,6 @@ array(
                     name: d
                 )
             )
-            comments: array(
-                0: // precedence
-            )
         )
         comments: array(
             0: // precedence

test/code/parser/expr/match.test

@@ -63,9 +63,6 @@ array(
                         conds: array(
                             0: Scalar_Int(
                                 value: 0
-                                comments: array(
-                                    0: // list of conditions
-                                )
                             )
                             1: Scalar_Int(
                                 value: 1

test/code/parser/expr/math.test

@@ -39,9 +39,6 @@ array(
             expr: Expr_Variable(
                 name: a
             )
-            comments: array(
-                0: // unary ops
-            )
         )
         comments: array(
             0: // unary ops
@@ -65,16 +62,10 @@ array(
         expr: Expr_BinaryOp_BitwiseAnd(
             left: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // binary ops
-                )
             )
             right: Expr_Variable(
                 name: b
             )
-            comments: array(
-                0: // binary ops
-            )
         )
         comments: array(
             0: // binary ops
@@ -195,23 +186,14 @@ array(
             left: Expr_BinaryOp_Mul(
                 left: Expr_Variable(
                     name: a
-                    comments: array(
-                        0: // associativity
-                    )
                 )
                 right: Expr_Variable(
                     name: b
                 )
-                comments: array(
-                    0: // associativity
-                )
             )
             right: Expr_Variable(
                 name: c
             )
-            comments: array(
-                0: // associativity
-            )
         )
         comments: array(
             0: // associativity
@@ -236,9 +218,6 @@ array(
         expr: Expr_BinaryOp_Plus(
             left: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // precedence
-                )
             )
             right: Expr_BinaryOp_Mul(
                 left: Expr_Variable(
@@ -248,9 +227,6 @@ array(
                     name: c
                 )
             )
-            comments: array(
-                0: // precedence
-            )
         )
         comments: array(
             0: // precedence
@@ -275,9 +251,6 @@ array(
         expr: Expr_BinaryOp_Pow(
             left: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // pow is special
-                )
             )
             right: Expr_BinaryOp_Pow(
                 left: Expr_Variable(
@@ -287,9 +260,6 @@ array(
                     name: c
                 )
             )
-            comments: array(
-                0: // pow is special
-            )
         )
         comments: array(
             0: // pow is special

test/code/parser/expr/new.test

@@ -52,9 +52,6 @@ array(
             )
             args: array(
             )
-            comments: array(
-                0: // class name variations
-            )
         )
         comments: array(
             0: // class name variations
@@ -100,9 +97,6 @@ array(
             )
             args: array(
             )
-            comments: array(
-                0: // DNCR object access
-            )
         )
         comments: array(
             0: // DNCR object access

test/code/parser/expr/ternaryAndCoalesce.test

@@ -21,9 +21,6 @@ array(
         expr: Expr_Ternary(
             cond: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // ternary
-                )
             )
             if: Expr_Variable(
                 name: b
@@ -31,9 +28,6 @@ array(
             else: Expr_Variable(
                 name: c
             )
-            comments: array(
-                0: // ternary
-            )
         )
         comments: array(
             0: // ternary
@@ -55,9 +49,6 @@ array(
             cond: Expr_Ternary(
                 cond: Expr_Variable(
                     name: a
-                    comments: array(
-                        0: // precedence
-                    )
                 )
                 if: Expr_Variable(
                     name: b
@@ -65,9 +56,6 @@ array(
                 else: Expr_Variable(
                     name: c
                 )
-                comments: array(
-                    0: // precedence
-                )
             )
             if: Expr_Variable(
                 name: d
@@ -75,9 +63,6 @@ array(
             else: Expr_Variable(
                 name: e
             )
-            comments: array(
-                0: // precedence
-            )
         )
         comments: array(
             0: // precedence
@@ -108,16 +93,10 @@ array(
         expr: Expr_BinaryOp_Coalesce(
             left: Expr_Variable(
                 name: a
-                comments: array(
-                    0: // null coalesce
-                )
             )
             right: Expr_Variable(
                 name: b
             )
-            comments: array(
-                0: // null coalesce
-            )
         )
         comments: array(
             0: // null coalesce

test/code/parser/formattingAttributes.test

@@ -0,0 +1,304 @@
+Test formatting attributes
+-----
+<?php
+
+0b1100;
+0o14;
+12;
+0xc;
+1_2_3_4_5_6;
+3.141_592_653;
+
+'foo';
+"bar";
+"foo
+bar";
+"foo\nbar";
+"foo\nbar{$x}";
+`foo\nbar`;
+`foo\nbar{$x}`;
+
+<<<'ABC'
+ABC;
+<<<'ABC'
+foo bar
+ABC;
+<<<'ABC'
+    foo bar
+    ABC;
+<<<ABC
+foo\nbar
+ABC;
+<<<ABC
+    foo\nbar
+    ABC;
+<<<ABC
+foo\nbar{$x}baz
+ABC;
+<<<ABC
+    foo\nbar{$x}baz
+    ABC;
+
+array();
+[];
+list($x) = $y;
+[$x] = $y;
+-----
+!!attributes
+array(
+    0: Stmt_Expression(
+        expr: Scalar_Int(
+            value: 12
+            rawValue: 0b1100
+            kind: KIND_BIN (2)
+        )
+    )
+    1: Stmt_Expression(
+        expr: Scalar_Int(
+            value: 12
+            rawValue: 0o14
+            kind: KIND_OCT (8)
+        )
+    )
+    2: Stmt_Expression(
+        expr: Scalar_Int(
+            value: 12
+            rawValue: 12
+            kind: KIND_DEC (10)
+        )
+    )
+    3: Stmt_Expression(
+        expr: Scalar_Int(
+            value: 12
+            rawValue: 0xc
+            kind: KIND_HEX (16)
+        )
+    )
+    4: Stmt_Expression(
+        expr: Scalar_Int(
+            value: 123456
+            rawValue: 1_2_3_4_5_6
+            kind: KIND_DEC (10)
+        )
+    )
+    5: Stmt_Expression(
+        expr: Scalar_Float(
+            value: 3.141592653
+            rawValue: 3.141_592_653
+        )
+    )
+    6: Stmt_Expression(
+        expr: Scalar_String(
+            value: foo
+            kind: KIND_SINGLE_QUOTED (1)
+            rawValue: 'foo'
+        )
+    )
+    7: Stmt_Expression(
+        expr: Scalar_String(
+            value: bar
+            kind: KIND_DOUBLE_QUOTED (2)
+            rawValue: "bar"
+        )
+    )
+    8: Stmt_Expression(
+        expr: Scalar_String(
+            value: foo
+            bar
+            kind: KIND_DOUBLE_QUOTED (2)
+            rawValue: "foo
+            bar"
+        )
+    )
+    9: Stmt_Expression(
+        expr: Scalar_String(
+            value: foo
+            bar
+            kind: KIND_DOUBLE_QUOTED (2)
+            rawValue: "foo\nbar"
+        )
+    )
+    10: Stmt_Expression(
+        expr: Scalar_InterpolatedString(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: foo
+                    bar
+                    rawValue: foo\nbar
+                )
+                1: Expr_Variable(
+                    name: x
+                )
+            )
+            kind: KIND_DOUBLE_QUOTED (2)
+        )
+    )
+    11: Stmt_Expression(
+        expr: Expr_ShellExec(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: foo
+                    bar
+                    rawValue: foo\nbar
+                )
+            )
+        )
+    )
+    12: Stmt_Expression(
+        expr: Expr_ShellExec(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: foo
+                    bar
+                    rawValue: foo\nbar
+                )
+                1: Expr_Variable(
+                    name: x
+                )
+            )
+        )
+    )
+    13: Stmt_Expression(
+        expr: Scalar_String(
+            value:
+            kind: KIND_NOWDOC (4)
+            docLabel: ABC
+            docIndentation:
+            rawValue:
+        )
+    )
+    14: Stmt_Expression(
+        expr: Scalar_String(
+            value: foo bar
+            kind: KIND_NOWDOC (4)
+            docLabel: ABC
+            docIndentation:
+            rawValue: foo bar
+        )
+    )
+    15: Stmt_Expression(
+        expr: Scalar_String(
+            value: foo bar
+            kind: KIND_NOWDOC (4)
+            docLabel: ABC
+            docIndentation:
+            rawValue: foo bar
+        )
+    )
+    16: Stmt_Expression(
+        expr: Scalar_String(
+            value: foo
+            bar
+            kind: KIND_HEREDOC (3)
+            docLabel: ABC
+            docIndentation:
+            rawValue: foo\nbar
+        )
+    )
+    17: Stmt_Expression(
+        expr: Scalar_String(
+            value: foo
+            bar
+            kind: KIND_HEREDOC (3)
+            docLabel: ABC
+            docIndentation:
+            rawValue: foo\nbar
+        )
+    )
+    18: Stmt_Expression(
+        expr: Scalar_InterpolatedString(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: foo
+                    bar
+                    rawValue: foo\nbar
+                )
+                1: Expr_Variable(
+                    name: x
+                )
+                2: InterpolatedStringPart(
+                    value: baz
+                    rawValue: baz
+                )
+            )
+            kind: KIND_HEREDOC (3)
+            docLabel: ABC
+            docIndentation:
+        )
+    )
+    19: Stmt_Expression(
+        expr: Scalar_InterpolatedString(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: foo
+                    bar
+                    rawValue: foo\nbar
+                )
+                1: Expr_Variable(
+                    name: x
+                )
+                2: InterpolatedStringPart(
+                    value: baz
+                    rawValue: baz
+                )
+            )
+            kind: KIND_HEREDOC (3)
+            docLabel: ABC
+            docIndentation:
+        )
+    )
+    20: Stmt_Expression(
+        expr: Expr_Array(
+            items: array(
+            )
+            kind: KIND_LONG (1)
+        )
+    )
+    21: Stmt_Expression(
+        expr: Expr_Array(
+            items: array(
+            )
+            kind: KIND_SHORT (2)
+        )
+    )
+    22: Stmt_Expression(
+        expr: Expr_Assign(
+            var: Expr_List(
+                items: array(
+                    0: ArrayItem(
+                        key: null
+                        value: Expr_Variable(
+                            name: x
+                        )
+                        byRef: false
+                        unpack: false
+                    )
+                )
+                kind: KIND_LIST (1)
+            )
+            expr: Expr_Variable(
+                name: y
+            )
+        )
+    )
+    23: Stmt_Expression(
+        expr: Expr_Assign(
+            var: Expr_List(
+                items: array(
+                    0: ArrayItem(
+                        key: null
+                        value: Expr_Variable(
+                            name: x
+                        )
+                        byRef: false
+                        unpack: false
+                    )
+                )
+                kind: KIND_ARRAY (2)
+            )
+            expr: Expr_Variable(
+                name: y
+            )
+        )
+    )
+)

test/code/parser/nopPositions.test

@@ -19,9 +19,13 @@ array(
 -----
 !!positions
 array(
-    0: Stmt_Nop[3:0 - 3:17](
-        comments: array(
-            0: /* comment */
+    0: Stmt_Block[2:1 - 4:1](
+        stmts: array(
+            0: Stmt_Nop[3:0 - 3:17](
+                comments: array(
+                    0: /* comment */
+                )
+            )
         )
     )
 )

test/code/parser/scalar/constantString.test

@@ -75,7 +75,7 @@ array(
     11: Stmt_Expression(
         expr: Scalar_String(
             value: !"!\!$!
-        !@@{ "\r" }@@!@@{ "\t" }@@!@@{ "\f" }@@!@@{ "\v" }@@!@@{ chr(27) /* "\e" */ }@@!\a
+            !@@{ "\r" }@@!@@{ "\t" }@@!@@{ "\f" }@@!@@{ "\v" }@@!@@{ chr(27) /* "\e" */ }@@!\a
         )
     )
     12: Stmt_Expression(
@@ -83,4 +83,4 @@ array(
             value: !@@{ chr(255) }@@!@@{ chr(255) }@@!@@{ chr(0) }@@!@@{ chr(0) }@@!
         )
     )
-)
+)

test/code/parser/scalar/docString.test

@@ -37,9 +37,6 @@ array(
     0: Stmt_Expression(
         expr: Scalar_String(
             value:
-            comments: array(
-                0: // empty strings
-            )
         )
         comments: array(
             0: // empty strings
@@ -53,9 +50,6 @@ array(
     2: Stmt_Expression(
         expr: Scalar_String(
             value: Test '" $a \n
-            comments: array(
-                0: // constant encapsed strings
-            )
         )
         comments: array(
             0: // constant encapsed strings
@@ -77,9 +71,6 @@ array(
                     name: a
                 )
             )
-            comments: array(
-                0: // encapsed strings
-            )
         )
         comments: array(
             0: // encapsed strings

test/code/parser/scalar/flexibleDocString.test

@@ -155,12 +155,12 @@ array(
     4: Stmt_Expression(
         expr: Scalar_String(
             value:     a
-           b
+               b
 
-          c
+              c
 
-         d
-        e
+             d
+            e
         )
     )
     5: Stmt_Expression(
@@ -168,7 +168,7 @@ array(
             parts: array(
                 0: InterpolatedStringPart(
                     value:     a
-                   b
+                       b
 
                 )
                 1: Expr_Variable(
@@ -176,8 +176,8 @@ array(
                 )
                 2: InterpolatedStringPart(
                     value:
-                 d
-                e
+                     d
+                    e
                 )
             )
         )
@@ -185,15 +185,15 @@ array(
     6: Stmt_Expression(
         expr: Scalar_String(
             value:
-            a
+                a
 
-           b
+               b
 
-          c
+              c
 
-         d
+             d
 
-        e
+            e
 
         )
     )
@@ -203,9 +203,9 @@ array(
                 0: InterpolatedStringPart(
                     value: 	a
 
-                @@{ "\t" }@@a
+                    @@{ "\t" }@@a
 
-                   b
+                       b
 
 
                 )
@@ -215,9 +215,9 @@ array(
                 2: InterpolatedStringPart(
                     value:
 
-                 d
+                     d
 
-                e
+                    e
 
                 )
             )
@@ -321,7 +321,7 @@ array(
                 )
                 1: InterpolatedStringPart(
                     value:
-                -
+                    -
                 )
             )
         )
@@ -334,7 +334,7 @@ array(
                 )
                 1: InterpolatedStringPart(
                     value:
-                 -
+                     -
                 )
             )
         )

test/code/parser/scalar/flexibleDocStringErrors.test

@@ -67,8 +67,8 @@ array(
         exprs: array(
             0: Scalar_String(
                 value:  a
-            b
-            c
+                b
+                c
             )
         )
     )

test/code/parser/scalar/float.test

@@ -76,10 +76,6 @@ array(
     10: Stmt_Expression(
         expr: Scalar_Float(
             value: 1.844674407371E+19
-            comments: array(
-                0: // various integer -> float overflows
-                1: // (all are actually the same number, just in different representations)
-            )
         )
         comments: array(
             0: // various integer -> float overflows

test/code/parser/scalar/numberSeparators.test

@@ -59,12 +59,6 @@ array(
         expr: Expr_ConstFetch(
             name: Name(
                 name: _100
-                comments: array(
-                    0: // already a valid constant name
-                )
-            )
-            comments: array(
-                0: // already a valid constant name
             )
         )
         comments: array(
@@ -74,9 +68,6 @@ array(
     6: Stmt_Expression(
         expr: Scalar_Int(
             value: 100
-            comments: array(
-                0: // syntax errors
-            )
         )
         comments: array(
             0: // syntax errors

test/code/parser/scalar/unicodeEscape.test

@@ -3,8 +3,8 @@ Unicode escape sequence
 <?php
 
 "\u{0}";
-"\u{114}";
-"\u{1F602}";
+"\u{114}$foo";
+`\u{1F602}$bar`;
 -----
 array(
     0: Stmt_Expression(
@@ -13,13 +13,66 @@ array(
         )
     )
     1: Stmt_Expression(
-        expr: Scalar_String(
-            value: Ĕ
+        expr: Scalar_InterpolatedString(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: Ĕ
+                )
+                1: Expr_Variable(
+                    name: foo
+                )
+            )
         )
     )
     2: Stmt_Expression(
+        expr: Expr_ShellExec(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: @@{"\xF0\x9F\x98\x82"}@@
+                )
+                1: Expr_Variable(
+                    name: bar
+                )
+            )
+        )
+    )
+)
+-----
+<?php
+
+"\u{0}";
+"\u{114}$foo";
+`\u{1F602}$bar`;
+-----
+!!version=5.6
+array(
+    0: Stmt_Expression(
         expr: Scalar_String(
-            value: @@{"\xF0\x9F\x98\x82"}@@
+            value: \u{0}
+        )
+    )
+    1: Stmt_Expression(
+        expr: Scalar_InterpolatedString(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: \u{114}
+                )
+                1: Expr_Variable(
+                    name: foo
+                )
+            )
+        )
+    )
+    2: Stmt_Expression(
+        expr: Expr_ShellExec(
+            parts: array(
+                0: InterpolatedStringPart(
+                    value: \u{1F602}
+                )
+                1: Expr_Variable(
+                    name: bar
+                )
+            )
         )
     )
 )

test/code/parser/stmt/class/name.test

@@ -42,6 +42,10 @@ array(
 -----
 Syntax error, unexpected T_STATIC, expecting T_STRING from 1:13 to 1:18
 array(
+    0: Stmt_Block(
+        stmts: array(
+        )
+    )
 )
 -----
 <?php class A extends self {}
@@ -211,6 +215,10 @@ array(
 -----
 Syntax error, unexpected T_STATIC, expecting T_STRING from 1:17 to 1:22
 array(
+    0: Stmt_Block(
+        stmts: array(
+        )
+    )
 )
 -----
 <?php interface A extends self {}

test/code/parser/stmt/class/readonlyAsClassName.test

@@ -5,6 +5,10 @@ class ReadOnly {}
 -----
 Syntax error, unexpected T_READONLY, expecting T_STRING from 2:7 to 2:14
 array(
+    0: Stmt_Block(
+        stmts: array(
+        )
+    )
 )
 -----
 <?php

test/code/parser/stmt/controlFlow.test

@@ -41,9 +41,11 @@ array(
             name: a
         )
     )
-    6: Stmt_Throw(
-        expr: Expr_Variable(
-            name: e
+    6: Stmt_Expression(
+        expr: Expr_Throw(
+            expr: Expr_Variable(
+                name: e
+            )
         )
     )
     7: Stmt_Label(

test/code/parser/stmt/declare.test

@@ -4,7 +4,9 @@ Declare
 
 declare (X='Y');
 
-declare (A='B', C='D') {}
+declare (A='B', C='D') {
+    echo "foo";
+}
 
 declare (A='B', C='D'):
 enddeclare;
@@ -43,6 +45,13 @@ array(
             )
         )
         stmts: array(
+            0: Stmt_Echo(
+                exprs: array(
+                    0: Scalar_String(
+                        value: foo
+                    )
+                )
+            )
         )
     )
     2: Stmt_Declare(

test/code/parser/stmt/function/typeVersions.test

@@ -83,9 +83,6 @@ array(
                 flags: 0
                 type: Name(
                     name: iterable
-                    comments: array(
-                        0: // PHP 7.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -103,9 +100,6 @@ array(
                 flags: 0
                 type: Name(
                     name: object
-                    comments: array(
-                        0: // PHP 7.1
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -123,9 +117,6 @@ array(
                 flags: 0
                 type: Name(
                     name: mixed
-                    comments: array(
-                        0: // PHP 7.2
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -143,9 +134,6 @@ array(
                 flags: 0
                 type: Name(
                     name: null
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -163,9 +151,6 @@ array(
                 flags: 0
                 type: Name(
                     name: false
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -292,9 +277,6 @@ array(
                 flags: 0
                 type: Name(
                     name: iterable
-                    comments: array(
-                        0: // PHP 7.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -312,9 +294,6 @@ array(
                 flags: 0
                 type: Name(
                     name: object
-                    comments: array(
-                        0: // PHP 7.1
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -332,9 +311,6 @@ array(
                 flags: 0
                 type: Name(
                     name: mixed
-                    comments: array(
-                        0: // PHP 7.2
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -352,9 +328,6 @@ array(
                 flags: 0
                 type: Name(
                     name: null
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -372,9 +345,6 @@ array(
                 flags: 0
                 type: Name(
                     name: false
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -501,9 +471,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: iterable
-                    comments: array(
-                        0: // PHP 7.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -521,9 +488,6 @@ array(
                 flags: 0
                 type: Name(
                     name: object
-                    comments: array(
-                        0: // PHP 7.1
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -541,9 +505,6 @@ array(
                 flags: 0
                 type: Name(
                     name: mixed
-                    comments: array(
-                        0: // PHP 7.2
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -561,9 +522,6 @@ array(
                 flags: 0
                 type: Name(
                     name: null
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -581,9 +539,6 @@ array(
                 flags: 0
                 type: Name(
                     name: false
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -710,9 +665,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: iterable
-                    comments: array(
-                        0: // PHP 7.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -730,9 +682,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: object
-                    comments: array(
-                        0: // PHP 7.1
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -750,9 +699,6 @@ array(
                 flags: 0
                 type: Name(
                     name: mixed
-                    comments: array(
-                        0: // PHP 7.2
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -770,9 +716,6 @@ array(
                 flags: 0
                 type: Name(
                     name: null
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -790,9 +733,6 @@ array(
                 flags: 0
                 type: Name(
                     name: false
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -919,9 +859,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: iterable
-                    comments: array(
-                        0: // PHP 7.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -939,9 +876,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: object
-                    comments: array(
-                        0: // PHP 7.1
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -959,9 +893,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: mixed
-                    comments: array(
-                        0: // PHP 7.2
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -979,9 +910,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: null
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -999,9 +927,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: false
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -1128,9 +1053,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: iterable
-                    comments: array(
-                        0: // PHP 7.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -1148,9 +1070,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: object
-                    comments: array(
-                        0: // PHP 7.1
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -1168,9 +1087,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: mixed
-                    comments: array(
-                        0: // PHP 7.2
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -1188,9 +1104,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: null
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false
@@ -1208,9 +1121,6 @@ array(
                 flags: 0
                 type: Identifier(
                     name: false
-                    comments: array(
-                        0: // PHP 8.0
-                    )
                 )
                 byRef: false
                 variadic: false

test/code/parser/stmt/generator/basic.test

@@ -47,9 +47,6 @@ array(
                 expr: Expr_Yield(
                     key: null
                     value: null
-                    comments: array(
-                        0: // statements
-                    )
                 )
                 comments: array(
                     0: // statements
@@ -77,17 +74,11 @@ array(
                 expr: Expr_Assign(
                     var: Expr_Variable(
                         name: data
-                        comments: array(
-                            0: // expressions
-                        )
                     )
                     expr: Expr_Yield(
                         key: null
                         value: null
                     )
-                    comments: array(
-                        0: // expressions
-                    )
                 )
                 comments: array(
                     0: // expressions
@@ -214,9 +205,6 @@ array(
                 expr: Expr_FuncCall(
                     name: Name(
                         name: func
-                        comments: array(
-                            0: // yield in function calls
-                        )
                     )
                     args: array(
                         0: Arg(
@@ -231,9 +219,6 @@ array(
                             unpack: false
                         )
                     )
-                    comments: array(
-                        0: // yield in function calls
-                    )
                 )
                 comments: array(
                     0: // yield in function calls

test/code/parser/stmt/haltCompilerOffset.test

@@ -28,6 +28,6 @@ array(
     )
     1: Stmt_HaltCompiler(
         remaining:
-    Foo
+        Foo
     )
 )

test/code/parser/stmt/namespace/groupUseErrors.test

@@ -63,17 +63,21 @@ array(
             0: // Missing NS separator
         )
     )
-    1: Stmt_Expression(
-        expr: Expr_ConstFetch(
-            name: Name(
-                name: Bar
+    1: Stmt_Block(
+        stmts: array(
+            0: Stmt_Expression(
+                expr: Expr_ConstFetch(
+                    name: Name(
+                        name: Bar
+                    )
+                )
             )
-        )
-    )
-    2: Stmt_Expression(
-        expr: Expr_ConstFetch(
-            name: Name(
-                name: Baz
+            1: Stmt_Expression(
+                expr: Expr_ConstFetch(
+                    name: Name(
+                        name: Baz
+                    )
+                )
             )
         )
     )

test/code/prettyPrinter/stmt/block.test

@@ -0,0 +1,28 @@
+Block statements
+-----
+<?php
+echo "a";
+{
+}
+{
+    echo "b";
+}
+if ($c) {
+    echo "c";
+    {
+        echo "d";
+    }
+}
+-----
+echo "a";
+{
+}
+{
+    echo "b";
+}
+if ($c) {
+    echo "c";
+    {
+        echo "d";
+    }
+}