Update documentation links in README

Fixes #967.

.gitattributes

@@ -1,5 +1,6 @@
 /.github         export-ignore
 /doc             export-ignore
+/grammar         export-ignore
 /test            export-ignore
 /test_old        export-ignore
 /tools           export-ignore

.github/workflows/main.yml

@@ -7,15 +7,15 @@ on:
 jobs:
   tests_coverage:
     runs-on: "ubuntu-latest"
-    name: "PHP 7.1 Unit Tests (with coverage)"
+    name: "PHP 7.4 Unit Tests (with coverage)"
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
           coverage: "xdebug"
-          php-version: "7.1"
+          php-version: "7.4"
           tools: composer:v2
       - name: "Install dependencies"
         run: |
@@ -34,15 +34,13 @@ jobs:
     strategy:
       matrix:
         php-version:
-          - "7.2"
-          - "7.3"
-          - "7.4"
           - "8.0"
           - "8.1"
           - "8.2"
+          - "8.3"
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
@@ -55,54 +53,67 @@ jobs:
         run: "php vendor/bin/phpunit"
   test_old_73_80:
     runs-on: "ubuntu-latest"
-    name: "PHP 7.3 Code on PHP 8.0 Integration Tests"
+    name: "PHP 7.4 Code on PHP 8.2 Integration Tests"
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
           coverage: "none"
-          php-version: "8.0"
+          php-version: "8.2"
           tools: composer:v2
       - name: "Install PHP 8 dependencies"
         run: "composer update --no-progress --prefer-dist"
       - name: "Tests"
-        run: "test_old/run-php-src.sh 7.3.21"
+        run: "test_old/run-php-src.sh 7.4.33"
   test_old_80_70:
     runs-on: "ubuntu-latest"
-    name: "PHP 8.2 Code on PHP 7.1 Integration Tests"
+    name: "PHP 8.3 Code on PHP 7.4 Integration Tests"
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
           coverage: "none"
-          php-version: "7.1"
+          php-version: "7.4"
           tools: composer:v2
       - name: "Install PHP 8 dependencies"
         run: "composer update --no-progress --prefer-dist"
       - name: "Tests"
-        run: "test_old/run-php-src.sh 8.2.3"
+        run: "test_old/run-php-src.sh 8.3.0RC2"
   phpstan:
     runs-on: "ubuntu-latest"
-    name: "PHP ${{ matrix.php-version }} PHPStan"
-    strategy:
-      matrix:
-        php-version:
-          - "8.2"
+    name: "PHPStan"
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v2"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
           coverage: "none"
-          php-version: "${{ matrix.php-version }}"
+          php-version: "8.2"
           tools: composer:v2
       - name: "Install dependencies"
         run: |
           cd tools && composer install
       - name: "PHPStan"
         run: "php tools/vendor/bin/phpstan"
+  php-cs-fixer:
+    runs-on: "ubuntu-latest"
+    name: "PHP-CS-Fixer"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v3"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "8.2"
+          tools: composer:v2
+      - name: "Install dependencies"
+        run: |
+          cd tools && composer install
+      - name: "php-cs-fixer"
+        run: "php tools/vendor/bin/php-cs-fixer fix --dry-run"

.php-cs-fixer.dist.php

@@ -21,6 +21,11 @@ return $config->setRiskyAllowed(true)
         'declare_strict_types' => true,
         // Keep argument formatting for now.
         'method_argument_space' => ['on_multiline' => 'ignore'],
+        'phpdoc_align' => ['align' => 'left'],
+        'phpdoc_trim' => true,
+        'no_empty_phpdoc' => true,
+        'no_superfluous_phpdoc_tags' => ['allow_mixed' => true],
+        'no_extra_blank_lines' => true,
     ])
     ->setFinder($finder)
 ;

CHANGELOG.md

@@ -1,3 +1,79 @@
+Version 5.0.0 (2024-01-07)
+--------------------------
+
+See UPGRADE-5.0 for detailed migration instructions.
+
+### Fixed
+
+* Fixed parent class of `PropertyItem` and `UseItem`.
+
+Version 5.0.0-rc1 (2023-12-20)
+------------------------------
+
+See UPGRADE-5.0 for detailed migration instructions.
+
+### Fixed
+
+* Fixed parsing of empty files.
+
+### Added
+
+* Added support for printing additional attributes (like `kind`) in `NodeDumper`.
+* Added `rawValue` attribute to `InterpolatedStringPart` and heredoc/nowdoc `String_`s, which
+  provides the original, unparsed value. It was previously only available for non-interpolated
+  single/double quoted strings.
+* Added `Stmt\Block` to represent `{}` code blocks. Previously, such code blocks were flattened
+  into the parent statements array. `Stmt\Block` will not be created for structures that are
+  typically used with code blocks, for example `if ($x) { $y; }` will be represented as previously,
+  while `if ($x) { { $x; } }` will have an extra `Stmt\Block` wrapper.
+
+### Changed
+
+* Use visitor to assign comments. This fixes the long-standing issue where comments were assigned
+  to all nodes sharing a starting position. Now only the outer-most node will hold the comments.
+* Don't parse unicode escape sequences when targeting PHP < 7.0.
+* Improve NodeDumper performance for large dumps.
+
+### Removed
+
+* Removed `Stmt\Throw_` node, use `Expr\Throw_` inside `Stmt\Expression` instead.
+* Removed `ParserFactory::create()`.
+
+Version 5.0.0-beta1 (2023-09-17)
+--------------------------------
+
+See UPGRADE-5.0 for detailed migration instructions.
+
+### Added
+
+* Visitors can now be passed directly to the `NodeTraverser` constructor. A separate call to
+  `addVisitor()` is no longer required.
+
+### Changed
+
+* The minimum host PHP version is now PHP 7.4. It is still possible to parse code from older
+  versions. Property types have been added where possible.
+* The `Lexer` no longer accepts options. `Lexer\Emulative` only accepts a `PhpVersion`. The
+  `startLexing()`, `getTokens()` and `handleHaltCompiler()` methods have been removed. Instead,
+  there is a single method `tokenize()` returning the tokens.
+* The `Parser::getLexer()` method has been replaced by `Parser::getTokens()`.
+* Attribute handling has been moved from the lexer to the parser, and is no longer configurable.
+  The comments, startLine, endLine, startTokenPos, endTokenPos, startFilePos, and endFilePos
+  attributes will always be added.
+* The pretty printer now defaults to PHP 7.4 as the target version.
+* The pretty printer now indents heredoc/nowdoc strings if the target version is >= 7.3
+  (flexible heredoc/nowdoc).
+
+### Removed
+
+* The deprecated `Comment::getLine()`, `Comment::getTokenPos()` and `Comment::getFilePos()` methods
+  have been removed. Use `Comment::getStartLine()`, `Comment::getStartTokenPos()` and
+  `Comment::getStartFilePos()` instead.
+
+### Deprecated
+
+* The `Node::getLine()` method has been deprecated. Use `Node::getStartLine()` instead.
+
 Version 5.0.0-alpha3 (2023-06-24)
 ---------------------------------
 

Makefile

@@ -0,0 +1,10 @@
+.PHONY: phpstan php-cs-fixer
+
+tools/vendor:
+	composer install -d tools
+
+phpstan: tools/vendor
+	tools/vendor/bin/phpstan
+
+php-cs-fixer: tools/vendor
+	tools/vendor/bin/php-cs-fixer fix

README.md

@@ -6,11 +6,9 @@ PHP Parser
 This is a PHP parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
-[Documentation for version 5.x][doc_master] (in development; for running on PHP >= 7.1; for parsing PHP 7.0 to PHP 8.2, with limited support for parsing PHP 5.x).
+[**Documentation for version 5.x**][doc_master] (current; for running on PHP >= 7.4; for parsing PHP 7.0 to PHP 8.3, with limited support for parsing PHP 5.x).
 
-[**Documentation for version 4.x**][doc_4_x] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 8.2).
-
-[Documentation for version 3.x][doc_3_x] (unsupported; for running on PHP >= 5.5; for parsing PHP 5.2 to PHP 7.2).
+[Documentation for version 4.x][doc_4_x] (supported; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 8.3).
 
 Features
 --------
@@ -213,9 +211,8 @@ Component documentation:
  * [AST builders](doc/component/AST_builders.markdown)
    * Fluent builders for AST nodes
  * [Lexer](doc/component/Lexer.markdown)
-   * Lexer options
-   * Token and file positions for nodes
-   * Custom attributes
+   * Emulation
+   * Tokens, positions and attributes
  * [Error handling](doc/component/Error_handling.markdown)
    * Column information for errors
    * Error recovery (parsing of syntactically incorrect code)

UPGRADE-5.0.md

@@ -3,7 +3,7 @@ Upgrading from PHP-Parser 4.x to 5.0
 
 ### PHP version requirements
 
-PHP-Parser now requires PHP 7.1 or newer to run. It is however still possible to *parse* code for older versions, while running on a newer version.
+PHP-Parser now requires PHP 7.4 or newer to run. It is however still possible to *parse* code for older versions, while running on a newer version.
 
 ### PHP 5 parsing support
 
@@ -13,6 +13,7 @@ In particular, if an older `PhpVersion` is specified, then:
 
  * For versions before PHP 7.0, `$foo =& new Bar()` assignments are allowed without error.
  * For versions before PHP 7.0, invalid octal literals `089` are allowed without error.
+ * For versions before PHP 7.0, unicode escape sequences `\u{123}` in strings are not parsed.
  * Type hints are interpreted as a class `Name` or as a built-in `Identifier` depending on PHP
    version, for example `int` is treated as a class name on PHP 5.6 and as a built-in on PHP 7.0.
 
@@ -27,16 +28,17 @@ The following symbols are affected by this removal:
  * The `PhpParser\Parser\Php5` class has been removed.
  * The `PhpParser\Parser\Multiple` class has been removed. While not strictly related to PHP 5 support, this functionality is no longer useful without it.
  * The `PhpParser\ParserFactory::ONLY_PHP5` and `PREFER_PHP5` options have been removed.
- * The `PhpParser\ParserFactory::PREFER_PHP7` option is now equivalent to `ONLY_PHP7`.
 
 ### Changes to the parser factory
 
-The `ParserFactory::create()` method is deprecated in favor of three new methods that provide more fine-grained control over the PHP version being targeted:
+The `ParserFactory::create()` method has been removed in favor of three new methods that provide more fine-grained control over the PHP version being targeted:
 
  * `createForNewestSupportedVersion()`: Use this if you don't know the PHP version of the code you're parsing. It's better to assume a too new version than a too old one.
  * `createForHostVersion()`: Use this if you're parsing code for the PHP version you're running on.
  * `createForVersion()`: Use this if you know the PHP version of the code you want to parse.
 
+The `createForNewestSupportedVersion()` and `creatForHostVersion()` are available since PHP-Parser 4.18.0, to allow libraries to support PHP-Parser 4 and 5 at the same time more easily.
+
 In all cases, the PHP version is a fairly weak hint that is only used on a best-effort basis. The parser will usually accept code for newer versions if it does not have any backwards-compatibility implications.
 
 For example, if you specify version `"8.0"`, then `class ReadOnly {}` is treated as a valid class declaration, while using `public readonly int $prop` will lead to a parse error. However, `final public const X = Y;` will be accepted in both cases.
@@ -45,7 +47,7 @@ For example, if you specify version `"8.0"`, then `class ReadOnly {}` is treated
 use PhpParser\ParserFactory;
 use PhpParser\PhpVersion;
 
-$factory = new ParserFactory;
+$factory = new ParserFactory();
 
 # Before
 $parser = $factory->create(ParserFactory::PREFER_PHP7);
@@ -61,6 +63,35 @@ $parser = $factory->create(ParserFactory::ONLY_PHP5);
 $parser = $factory->createForVersion(PhpVersion::fromString("5.6"));
 ```
 
+### Changes to the throw representation
+
+Previously, `throw` statements like `throw $e;` were represented using the `Stmt\Throw_` class,
+while uses inside other expressions (such as `$x ?? throw $e`) used the `Expr\Throw_` class.
+
+Now, `throw $e;` is represented as a `Stmt\Expression` that contains an `Expr\Throw_`. The
+`Stmt\Throw_` class has been removed.
+
+```php
+# Code
+throw $e;
+
+# Before
+Stmt_Throw(
+    expr: Expr_Variable(
+        name: e
+    )
+)
+
+# After
+Stmt_Expression(
+    expr: Expr_Throw(
+        expr: Expr_Variable(
+            name: e
+        )
+    )
+)
+```
+
 ### Changes to the array destructuring representation
 
 Previously, the `list($x) = $y` destructuring syntax was represented using a `Node\Expr\List_`
@@ -71,6 +102,49 @@ Now, destructuring is always represented using `Node\Expr\List_`. The `kind` att
 `Node\Expr\List_::KIND_LIST` or `Node\Expr\List_::KIND_ARRAY` specifies which syntax was actually
 used.
 
+```php
+# Code
+[$x] = $y;
+
+# Before
+Expr_Assign(
+   var: Expr_Array(
+       items: array(
+           0: Expr_ArrayItem(
+               key: null
+               value: Expr_Variable(
+                   name: x
+               )
+               byRef: false
+               unpack: false
+           )
+       )
+   )
+   expr: Expr_Variable(
+       name: y
+   )
+)
+
+# After
+Expr_Assign(
+   var: Expr_List(
+       items: array(
+           0: ArrayItem(
+               key: null
+               value: Expr_Variable(
+                   name: x
+               )
+               byRef: false
+               unpack: false
+           )
+       )
+   )
+   expr: Expr_Variable(
+       name: y
+   )
+)
+```
+
 ### Changes to the name representation
 
 Previously, `Name` nodes had a `parts` subnode, which stores an array of name parts, split by
@@ -82,6 +156,123 @@ now represented by `Name(name: 'Foo\Bar')` instead.
 It is possible to convert the name to the previous representation using `$name->getParts()`. The
 `Name` constructor continues to accept both the string and the array representation.
 
+The `Name::getParts()` method is available since PHP-Parser 4.16.0, to allow libraries to support
+PHP-Parser 4 and 5 at the same time more easily.
+
+### Changes to the block representation
+
+Previously, code blocks `{ ... }` were always flattened into their parent statement list. For
+example `while ($x) { $a; { $b; } $c; }` would produce the same node structure as
+`if ($x) { $a; $b; $c; }`, namely a `Stmt\While_` node whose `stmts` subnode is an array of three
+statements.
+
+Now, the nested `{ $b; }` block is represented using an explicit `Stmt\Block` node. However, the
+outer `{ $a; { $b; } $c; }` block is still represented using a simple array in the `stmts` subnode.
+
+```php
+# Code
+while ($x) { $a; { $b; } $c; }
+
+# Before
+Stmt_While(
+    cond: Expr_Variable(
+        name: x
+    )
+    stmts: array(
+        0: Stmt_Expression(
+            expr: Expr_Variable(
+                name: a
+            )
+        )
+        1: Stmt_Expression(
+            expr: Expr_Variable(
+                name: b
+            )
+        )
+        2: Stmt_Expression(
+            expr: Expr_Variable(
+                name: c
+            )
+        )
+    )
+)
+
+# After
+Stmt_While(
+    cond: Expr_Variable(
+        name: x
+    )
+    stmts: array(
+        0: Stmt_Expression(
+            expr: Expr_Variable(
+                name: a
+            )
+        )
+        1: Stmt_Block(
+            stmts: array(
+                0: Stmt_Expression(
+                    expr: Expr_Variable(
+                        name: b
+                    )
+                )
+            )
+        )
+        2: Stmt_Expression(
+            expr: Expr_Variable(
+                name: c
+            )
+        )
+    )
+)
+```
+
+### Changes to comment assignment
+
+Previously, comments were assigned to all nodes starting at the same position. Now they will be
+assigned to the outermost node only.
+
+```php
+# Code
+// Comment
+$a + $b;
+
+# Before
+Stmt_Expression(
+    expr: Expr_BinaryOp_Plus(
+        left: Expr_Variable(
+            name: a
+            comments: array(
+                0: // Comment
+            )
+        )
+        right: Expr_Variable(
+            name: b
+        )
+        comments: array(
+            0: // Comment
+        )
+    )
+    comments: array(
+        0: // Comment
+    )
+)
+
+# After
+Stmt_Expression(
+    expr: Expr_BinaryOp_Plus(
+        left: Expr_Variable(
+            name: a
+        )
+        right: Expr_Variable(
+            name: b
+        )
+    )
+    comments: array(
+        0: // Comment
+    )
+)
+```
+
 ### Renamed nodes
 
 A number of AST nodes have been renamed or moved in the AST hierarchy:
@@ -104,7 +295,7 @@ The old class names have been retained as aliases for backwards compatibility. H
 
 Modifier flags (as used by the `$flags` subnode of `Class_`, `ClassMethod`, `Property`, etc.) are now available as class constants on a separate `PhpParser\Modifiers` class, instead of being part of `PhpParser\Node\Stmt\Class_`, to make it clearer that these are used by many different nodes. The old constants are deprecated, but are still available.
 
-```
+```php
 PhpParser\Node\Stmt\Class_::MODIFIER_PUBLIC    -> PhpParser\Modifiers::PUBLIC
 PhpParser\Node\Stmt\Class_::MODIFIER_PROTECTED -> PhpParser\Modifiers::PROTECTED
 PhpParser\Node\Stmt\Class_::MODIFIER_PRIVATE   -> PhpParser\Modifiers::PRIVATE
@@ -126,7 +317,7 @@ Node constructor arguments accepting types now longer accept plain strings. Eith
 * The `$type` argument of `Node\NullableType`.
 * The `$type` argument of `Node\Param`.
 * The `$type` argument of `Node\Stmt\Property`.
-* The `$type` argument of `Node\ClassConst` (new in PHP-Parser 5.0, listed for completeness only).
+* The `$type` argument of `Node\ClassConst`.
 
 To follow the previous behavior, an `Identifier` should be passed, which indicates a built-in type.
 
@@ -198,12 +389,13 @@ else if ($x) {
 }
 ```
 
-The pretty printer now accepts a `phpVersion` option, which accepts a `PhpVersion` object and defaults to PHP 7.1. The pretty printer will make formatting choices to make the code valid for that version. It currently controls the following behavior:
+The pretty printer now accepts a `phpVersion` option, which accepts a `PhpVersion` object and defaults to PHP 7.4. The pretty printer will make formatting choices to make the code valid for that version. It currently controls the following behavior:
 
 * For PHP >= 7.0 (default), short array syntax `[]` will be used by default. This does not affect nodes that specify an explicit array syntax using the `kind` attribute.
 * For PHP >= 7.0 (default), parentheses around `yield` expressions will only be printed when necessary. Previously, parentheses were always printed, even if `yield` was used as a statement.
-* For PHP >= 7.1 (default), the short array syntax `[]` will be used for destructuring by default (instead of `list()`). This does not affect nodes that specify and explicit syntax using the `kind` attribute.
-* For PHP >= 7.3, a newline is no longer forced after heredoc/nowdoc strings, as the requirement for this has been removed with the introduction of flexible heredoc/nowdoc strings.
+* For PHP >= 7.1 (default), the short array syntax `[]` will be used for destructuring by default (instead of `list()`). This does not affect nodes that specify an explicit syntax using the `kind` attribute.
+* For PHP >= 7.3 (default), a newline is no longer forced after heredoc/nowdoc strings, as the requirement for this has been removed with the introduction of flexible heredoc/nowdoc strings.
+* For PHP >= 7.3 (default), heredoc/nowdoc strings are now indented just like regular code. This was allowed with the introduction of flexible heredoc/nowdoc strings.
 
 ### Changes to precedence handling in the pretty printer
 
@@ -219,7 +411,7 @@ protected function p(
 
 The `$precedence` is the precedence of the direct parent operator (if any), while `$lhsPrecedence` is that precedence of the nearest binary operator on whose left-hand-side the node occurs. For unary operators, only the `$lhsPrecedence` is relevant.
 
-Recursive calls in pretty-printer methods should generally continue calling `p()` without additional parameters. However, pretty-printer methods for operators that participate in precedence resolution need to be adjusted. For example, typical implementations for operators looks as follows now:
+Recursive calls in pretty-printer methods should generally continue calling `p()` without additional parameters. However, pretty-printer methods for operators that participate in precedence resolution need to be adjusted. For example, typical implementations for operators look as follows now:
 
 ```php
 protected function pExpr_BinaryOp_Plus(
@@ -265,6 +457,17 @@ PhpParser\NodeTraverser::DONT_TRAVERSE_CURRENT_AND_CHILDREN -> PhpParser\NodeVis
 PhpParser\NodeTraverser::STOP_TRAVERSAL -> PhpParser\NodeVisitor::STOP_TRAVERSAL
 ```
 
+Visitors can now also be passed directly to the `NodeTraverser` constructor:
+
+```php
+# Before (and still supported)
+$traverser = new NodeTraverser();
+$traverser->addVisitor(new NameResolver());
+
+# After
+$traverser = new NodeTraverser(new NameResolver());
+```
+
 ### Changes to token representation
 
 Tokens are now internally represented using the `PhpParser\Token` class, which exposes the same base interface as
@@ -282,11 +485,50 @@ class Token {
 }
 ```
 
-The `Lexer::getTokens()` method will now return an array of `Token`s, rather than an array of arrays and strings.
+The token array is now an array of `Token`s, rather than an array of arrays and strings.
 Additionally, the token array is now terminated by a sentinel token with ID 0.
 
+### Changes to the lexer
+
+The lexer API is reduced to a single `Lexer::tokenize()` method, which returns an array of tokens. The `startLexing()` and `getNextToken()` methods have been removed.
+
+Responsibility for determining start and end attributes for nodes has been moved from the lexer to the parser. The lexer no longer accepts an options array. The `usedAttributes` option has been removed without replacement, and the parser will now unconditionally add the `comments`, `startLine`, `endLine`, `startFilePos`, `endFilePos`, `startTokenPos` and `endTokenPos` attributes.
+
+There should no longer be a need to directly interact with the `Lexer` for end users, as the `ParserFactory` will create an appropriate instance, and no additional configuration of the lexer is necessary. To use formatting-preserving pretty printing, the setup boilerplate changes as follows:
+
+```php
+# Before
+
+$lexer = new Lexer\Emulative([
+    'usedAttributes' => [
+        'comments',
+        'startLine', 'endLine',
+        'startTokenPos', 'endTokenPos',
+    ],
+]);
+
+$parser = new Parser\Php7($lexer);
+$oldStmts = $parser->parse($code);
+$oldTokens = $lexer->getTokens();
+
+$traverser = new NodeTraverser();
+$traverser->addVisitor(new NodeVisitor\CloningVisitor());
+$newStmts = $traverser->traverse($oldStmts);
+
+# After
+
+$parser = (new ParserFactory())->createForNewestSupportedVersion();
+$oldStmts = $parser->parse($code);
+$oldTokens = $lexer->getTokens();
+
+$traverser = new NodeTraverser(new NodeVisitor\CloningVisitor());
+$newStmts = $traverser->traverse($oldStmts);
+```
+
 ### Miscellaneous changes
 
  * The deprecated `Builder\Param::setTypeHint()` method has been removed in favor of `Builder\Param::setType()`.
  * The deprecated `Error` constructor taking a start line has been removed. Pass `['startLine' => $startLine]` attributes instead.
+ * The deprecated `Comment::getLine()`, `Comment::getTokenPos()` and `Comment::getFilePos()` methods have been removed. Use `Comment::getStartLine()`, `Comment::getStartTokenPos()` and `Comment::getStartFilePos()` instead.
  * `Comment::getReformattedText()` now normalizes CRLF newlines to LF newlines.
+ * The `Node::getLine()` method has been deprecated. Use `Node::getStartLine()` instead.

bin/php-parse

@@ -26,10 +26,7 @@ if (empty($files)) {
     showHelp("Must specify at least one file.");
 }
 
-$lexerOptions = ['usedAttributes' => [
-    'startLine', 'endLine', 'startFilePos', 'endFilePos', 'comments'
-]];
-$parser = (new PhpParser\ParserFactory())->createForVersion($attributes['version'], $lexerOptions);
+$parser = (new PhpParser\ParserFactory())->createForVersion($attributes['version']);
 $dumper = new PhpParser\NodeDumper([
     'dumpComments' => true,
     'dumpPositions' => $attributes['with-positions'],

composer.json

@@ -13,7 +13,7 @@
         }
     ],
     "require": {
-        "php": ">=7.1",
+        "php": ">=7.4",
         "ext-tokenizer": "*",
         "ext-json": "*",
         "ext-ctype": "*"

doc/0_Introduction.markdown

@@ -13,14 +13,14 @@ application dealing with code programmatically. A parser constructs an [Abstract
 There are other ways of processing source code. One that PHP supports natively is using the
 token stream generated by [`token_get_all`][2]. The token stream is much more low level than
 the AST and thus has different applications: It allows to also analyze the exact formatting of
-a file. On the other hand the token stream is much harder to deal with for more complex analysis.
+a file. On the other hand, the token stream is much harder to deal with for more complex analysis.
 For example, an AST abstracts away the fact that, in PHP, variables can be written as `$foo`, but also
 as `$$bar`, `${'foobar'}` or even `${!${''}=barfoo()}`. You don't have to worry about recognizing
 all the different syntaxes from a stream of tokens.
 
 Another question is: Why would I want to have a PHP parser *written in PHP*? Well, PHP might not be
 a language especially suited for fast parsing, but processing the AST is much easier in PHP than it
-would be in other, faster languages like C. Furthermore the people most probably wanting to do
+would be in other, faster languages like C. Furthermore the people most likely wanting to do
 programmatic PHP code analysis are incidentally PHP developers, not C developers.
 
 What can it parse?
@@ -43,7 +43,7 @@ following caveats:
 
 As the parser is based on the tokens returned by `token_get_all` (which is only able to lex the PHP
 version it runs on), additionally a wrapper for emulating tokens from newer versions is provided.
-This allows to parse PHP 8.0 source code running on PHP 7.1, for example. This emulation is not
+This allows to parse PHP 8.3 source code running on PHP 7.4, for example. This emulation is not
 perfect, but works well in practice.
 
 Finally, it should be noted that the parser aims to accept all valid code, not reject all invalid
@@ -81,16 +81,16 @@ However, it does retain accurate position information, which can be used to insp
 What else can it do?
 --------------------
 
-Apart from the parser itself this package also bundles support for some other, related features:
+Apart from the parser itself, this package also bundles support for some other, related features:
 
  * Support for pretty printing, which is the act of converting an AST into PHP code. Please note
    that "pretty printing" does not imply that the output is especially pretty. It's just how it's
    called ;)
- * Support for serializing and unserializing the node tree to JSON
+ * Support for serializing and unserializing the node tree to JSON.
  * Support for dumping the node tree in a human-readable form (see the section above for an
-   example of how the output looks like)
- * Infrastructure for traversing and changing the AST (node traverser and node visitors)
- * A node visitor for resolving namespaced names
+   example of how the output looks like).
+ * Infrastructure for traversing and changing the AST (node traverser and node visitors).
+ * A node visitor for resolving namespaced names.
 
  [0]: http://en.wikipedia.org/wiki/Static_program_analysis
  [1]: http://en.wikipedia.org/wiki/Abstract_syntax_tree

doc/2_Usage_of_basic_components.markdown

@@ -50,7 +50,7 @@ The `createXYZ()` methods optionally accept an array of lexer options. Some use
 customized lexer options are discussed in the [lexer documentation](component/Lexer.markdown).
 
 Subsequently, you can pass PHP code (including the opening `<?php` tag) to the `parse()` method in
-order to create a syntax tree. If a syntax error is encountered, an `PhpParser\Error` exception will
+order to create a syntax tree. If a syntax error is encountered, a `PhpParser\Error` exception will
 be thrown by default:
 
 ```php
@@ -72,7 +72,7 @@ try {
     $stmts = $parser->parse($code);
     // $stmts is an array of statement nodes
 } catch (Error $e) {
-    echo 'Parse Error: ', $e->getMessage();
+    echo 'Parse Error: ', $e->getMessage(), "\n";
 }
 ```
 
@@ -206,11 +206,13 @@ without the `PhpParser\Node\` prefix and `\` replaced with `_`. It also does not
 It is possible to associate custom metadata with a node using the `setAttribute()` method. This data
 can then be retrieved using `hasAttribute()`, `getAttribute()` and `getAttributes()`.
 
-By default the lexer adds the `startLine`, `endLine` and `comments` attributes. `comments` is an array
-of `PhpParser\Comment[\Doc]` instances.
+By default, the parser adds the `startLine`, `endLine`, `startTokenPos`, `endTokenPos`,
+`startFilePos`, `endFilePos` and `comments` attributes. `comments` is an array of
+`PhpParser\Comment[\Doc]` instances.
 
-The start line can also be accessed using `getStartLine()` (instead of `getAttribute('startLine')`).
-The last doc comment from the `comments` attribute can be obtained using `getDocComment()`.
+The pre-defined attributes can also be accessed using `getStartLine()` instead of
+`getAttribute('startLine')`, and so on. The last doc comment from the `comments` attribute can be
+obtained using `getDocComment()`.
 
 Pretty printer
 --------------
@@ -244,7 +246,7 @@ try {
 
     echo $code;
 } catch (Error $e) {
-    echo 'Parse Error: ', $e->getMessage();
+    echo 'Parse Error: ', $e->getMessage(), "\n";
 }
 ```
 
@@ -252,7 +254,7 @@ The above code will output:
 
     echo 'Hello ', hi\getTarget();
 
-As you can see the source code was first parsed using `PhpParser\Parser->parse()`, then changed and then
+As you can see, the source code was first parsed using `PhpParser\Parser->parse()`, then changed and then
 again converted to code using `PhpParser\PrettyPrinter\Standard->prettyPrint()`.
 
 The `prettyPrint()` method pretty prints a statements array. It is also possible to pretty print only a
@@ -393,7 +395,7 @@ declarations that contains the namespaced name instead of only the shortname tha
 Example: Converting namespaced code to pseudo namespaces
 --------------------------------------------------------
 
-A small example to understand the concept: We want to convert namespaced code to pseudo namespaces
+A small example to understand the concept: We want to convert namespaced code to pseudo namespaces,
 so it works on 5.2, i.e. names like `A\\B` should be converted to `A_B`. Note that such conversions
 are fairly complicated if you take PHP's dynamic features into account, so our conversion will
 assume that no dynamic features are used.
@@ -445,7 +447,7 @@ foreach ($files as $file) {
 }
 ```
 
-Now lets start with the main code, the `NodeVisitor\NamespaceConverter`. One thing it needs to do
+Now lets start with the main code, the `NamespaceConverter`. One thing it needs to do
 is convert `A\\B` style names to `A_B` style ones.
 
 ```php

doc/README.md

@@ -27,9 +27,8 @@ Component documentation
   * [AST builders](component/AST_builders.markdown)
     * Fluent builders for AST nodes
   * [Lexer](component/Lexer.markdown)
-    * Lexer options
-    * Token and file positions for nodes
-    * Custom attributes
+    * Emulation
+    * Tokens, positions and attributes
   * [Error handling](component/Error_handling.markdown)
     * Column information for errors
     * Error recovery (parsing of syntactically incorrect code)

doc/component/AST_builders.markdown

@@ -15,9 +15,10 @@ accessed through `getNode()`. Fluent builders are available for
 the following syntactic elements:
 
  * namespaces and use statements
- * classes, interfaces and traits
+ * classes, interfaces, traits and enums
  * methods, functions and parameters
- * properties
+ * properties, class constants and enum cases
+ * trait uses and trait use adaptations
 
 Here is an example:
 
@@ -95,13 +96,13 @@ abstract class SomeOtherClass extends SomeClass implements A\Few, \Interfaces
         AnotherTrait::func insteadof SecondTrait;
     }
     protected $someProperty;
-    private $anotherProperty = array(1, 2, 3);
+    private $anotherProperty = [1, 2, 3];
     /**
      * This method does something.
      *
      * @param SomeClass And takes a parameter
      */
-    public abstract function someMethod(SomeClass $someParam): bool;
+    abstract public function someMethod(SomeClass $someParam): bool;
     protected function anotherMethod($someParam = 'test')
     {
         print $someParam;
@@ -133,6 +134,8 @@ nodes. The following methods are currently available:
  * `propertyFetch($var, $name)`: Creates a property fetch node. Converts `$name` to an `Identifier`
    node.
  * `concat(...$exprs)`: Create a tree of `BinaryOp\Concat` nodes for the given expressions.
+ * `attribute($name, $args)`: Create a `Attribute` node. Converts `$name` to a `Name` node and
+   normalizes arguments.
 
 These methods may be expanded on an as-needed basis. Please open an issue or PR if a common
 operation is missing.

doc/component/Constant_expression_evaluation.markdown

@@ -70,6 +70,7 @@ expressions, apart from the following:
  * `Expr\ConstFetch` (only null/false/true are handled)
  * `Expr\ClassConstFetch`
  * `Expr\New_` (since PHP 8.1)
+ * `Expr\PropertyFetch` (since PHP 8.2)
 
 Handling these expression types requires non-local information, such as which global constants are
 defined. By default, the evaluator will throw a `ConstExprEvaluationException` when it encounters
@@ -84,7 +85,7 @@ specifying an evaluation fallback function:
 use PhpParser\{ConstExprEvaluator, ConstExprEvaluationException};
 use PhpParser\Node\Expr;
 
-$evalutator = new ConstExprEvaluator(function(Expr $expr) {
+$evaluator = new ConstExprEvaluator(function(Expr $expr) {
     if ($expr instanceof Expr\ConstFetch) {
         return fetchConstantSomehow($expr);
     }
@@ -97,7 +98,7 @@ $evalutator = new ConstExprEvaluator(function(Expr $expr) {
 });
 
 try {
-    $evalutator->evaluateSilently($someExpr);
+    $evaluator->evaluateSilently($someExpr);
 } catch (ConstExprEvaluationException $e) {
     // Handle exception
 }

doc/component/Error_handling.markdown

@@ -4,29 +4,12 @@ Error handling
 Errors during parsing or analysis are represented using the `PhpParser\Error` exception class. In addition to an error
 message, an error can also store additional information about the location the error occurred at.
 
-How much location information is available depends on the origin of the error and how many lexer attributes have been
-enabled. At a minimum the start line of the error is usually available.
+How much location information is available depends on the origin of the error. At a minimum the start line of the error
+is usually available.
 
 Column information
 ------------------
 
-In order to receive information about not only the line, but also the column span an error occurred at, the file
-position attributes in the lexer need to be enabled:
-
-```php
-$lexerOptions = array(
-    'usedAttributes' => array('comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'),
-);
-$parser = (new PhpParser\ParserFactory())->createForHostVersion($lexerOptions);
-
-try {
-    $stmts = $parser->parse($code);
-    // ...
-} catch (PhpParser\Error $e) {
-    // ...
-}
-```
-
 Before using column information, its availability needs to be checked with `$e->hasColumnInfo()`, as the precise
 location of an error cannot always be determined. The methods for retrieving column information also have to be passed
 the source code of the parsed file. An example for printing an error:

doc/component/FAQ.markdown

@@ -16,8 +16,7 @@ use PhpParser\ParserFactory;
 
 $code = '...';
 
-$traverser = new NodeTraverser;
-$traverser->addVisitor(new ParentConnectingVisitor);
+$traverser = new NodeTraverser(new ParentConnectingVisitor);
 
 $parser = (new ParserFactory())->createForHostVersion();
 $ast    = $parser->parse($code);
@@ -39,8 +38,7 @@ use PhpParser\ParserFactory;
 
 $code = '...';
 
-$traverser = new NodeTraverser;
-$traverser->addVisitor(new NodeConnectingVisitor);
+$traverser = new NodeTraverser(new NodeConnectingVisitor);
 
 $parser = (new ParserFactory())->createForHostVersion();
 $ast    = $parser->parse($code);

doc/component/JSON_representation.markdown

@@ -35,65 +35,6 @@ This will result in the following output (which includes attributes):
 [
     {
         "nodeType": "Stmt_Function",
-        "byRef": false,
-        "name": {
-            "nodeType": "Identifier",
-            "name": "printLine",
-            "attributes": {
-                "startLine": 4,
-                "endLine": 4
-            }
-        },
-        "params": [
-            {
-                "nodeType": "Param",
-                "type": null,
-                "byRef": false,
-                "variadic": false,
-                "var": {
-                    "nodeType": "Expr_Variable",
-                    "name": "msg",
-                    "attributes": {
-                        "startLine": 4,
-                        "endLine": 4
-                    }
-                },
-                "default": null,
-                "attributes": {
-                    "startLine": 4,
-                    "endLine": 4
-                }
-            }
-        ],
-        "returnType": null,
-        "stmts": [
-            {
-                "nodeType": "Stmt_Echo",
-                "exprs": [
-                    {
-                        "nodeType": "Expr_Variable",
-                        "name": "msg",
-                        "attributes": {
-                            "startLine": 5,
-                            "endLine": 5
-                        }
-                    },
-                    {
-                        "nodeType": "Scalar_String",
-                        "value": "\n",
-                        "attributes": {
-                            "startLine": 5,
-                            "endLine": 5,
-                            "kind": 2
-                        }
-                    }
-                ],
-                "attributes": {
-                    "startLine": 5,
-                    "endLine": 5
-                }
-            }
-        ],
         "attributes": {
             "startLine": 4,
             "comments": [
@@ -101,12 +42,79 @@ This will result in the following output (which includes attributes):
                     "nodeType": "Comment_Doc",
                     "text": "\/** @param string $msg *\/",
                     "line": 3,
-                    "filePos": 9,
-                    "tokenPos": 2
+                    "filePos": 7,
+                    "tokenPos": 2,
+                    "endLine": 3,
+                    "endFilePos": 31,
+                    "endTokenPos": 2
                 }
             ],
             "endLine": 6
+        },
+        "byRef": false,
+        "name": {
+            "nodeType": "Identifier",
+            "attributes": {
+                "startLine": 4,
+                "endLine": 4
+            },
+            "name": "printLine"
+        },
+        "params": [
+            {
+                "nodeType": "Param",
+                "attributes": {
+                    "startLine": 4,
+                    "endLine": 4
+                },
+                "type": null,
+                "byRef": false,
+                "variadic": false,
+                "var": {
+                    "nodeType": "Expr_Variable",
+                    "attributes": {
+                        "startLine": 4,
+                        "endLine": 4
+                    },
+                    "name": "msg"
+                },
+                "default": null,
+                "flags": 0,
+                "attrGroups": []
             }
+        ],
+        "returnType": null,
+        "stmts": [
+            {
+                "nodeType": "Stmt_Echo",
+                "attributes": {
+                    "startLine": 5,
+                    "endLine": 5
+                },
+                "exprs": [
+                    {
+                        "nodeType": "Expr_Variable",
+                        "attributes": {
+                            "startLine": 5,
+                            "endLine": 5
+                        },
+                        "name": "msg"
+                    },
+                    {
+                        "nodeType": "Scalar_String",
+                        "attributes": {
+                            "startLine": 5,
+                            "endLine": 5
+                            "kind": 2,
+                            "rawValue": "\"\\n\""
+                        },
+                        "value": "\n"
+                    }
+                ]
+            }
+        ],
+        "attrGroups": [],
+        "namespacedName": null
     }
 ]
 ```

doc/component/Lexer.markdown

@@ -1,41 +1,79 @@
 Lexer component documentation
 =============================
 
-The lexer is responsible for providing tokens to the parser. The project comes with two lexers: `PhpParser\Lexer` and
-`PhpParser\Lexer\Emulative`. The latter is an extension of the former, which adds the ability to emulate tokens of
-newer PHP versions and thus allows parsing of new code on older versions.
+The lexer is responsible for providing tokens to the parser. Typical use of the library does not require direct
+interaction with the lexer, as an appropriate lexer is created by `PhpParser\ParserFactory`. The tokens produced
+by the lexer can then be retrieved using `PhpParser\Parser::getTokens()`.
 
-This documentation discusses options available for the default lexers and explains how lexers can be extended.
+Emulation
+---------
 
-Lexer options
--------------
+While this library implements a custom parser, it relies on PHP's `ext/tokenizer` extension to perform lexing. However,
+this extension only supports lexing code for the PHP version you are running on, while this library also wants to support
+parsing newer code. For that reason, the lexer performs additional "emulation" in three layers:
 
-The two default lexers accept an `$options` array in the constructor. Currently only the `'usedAttributes'` option is
-supported, which allows you to specify which attributes will be added to the AST nodes. The attributes can then be
-accessed using `$node->getAttribute()`, `$node->setAttribute()`, `$node->hasAttribute()` and `$node->getAttributes()`
-methods. A sample options array:
+First, PhpParser uses the `PhpToken` based representation introduced in PHP 8.0, rather than the array-based tokens from
+previous versions. The `PhpParser\Token` class either extends `PhpToken` (on PHP 8.0) or a polyfill implementation. The
+polyfill implementation will also perform two emulations that are required by the parser and cannot be disabled:
+
+ * Single-line comments use the PHP 8.0 representation that does not include a trailing newline. The newline will be
+   part of a following `T_WHITESPACE` token.
+ * Namespaced names use the PHP 8.0 representation using `T_NAME_FULLY_QUALIFIED`, `T_NAME_QUALIFIED` and
+   `T_NAME_RELATIVE` tokens, rather than the previous representation using a sequence of `T_STRING` and `T_NS_SEPARATOR`.
+   This means that certain code that is legal on older versions (namespaced names including whitespace, such as `A \ B`)
+   will not be accepted by the parser.
+
+Second, the `PhpParser\Lexer` base class will convert `&` tokens into the PHP 8.1 representation of either
+`T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG` or `T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG`. This is required by the parser
+and cannot be disabled.
+
+Finally, `PhpParser\Lexer\Emulative` performs other, optional emulations. This lexer is parameterized by `PhpVersion`
+and will try to emulate `ext/tokenizer` output for that version. This is done using separate `TokenEmulator`s for each
+emulated feature.
+
+Emulation is usually used to support newer PHP versions, but there is also very limited support for reverse emulation to
+older PHP versions, which can make keywords from newer versions non-reserved.
+
+Tokens, positions and attributes
+--------------------------------
+
+The `Lexer::tokenize()` method returns an array of `PhpParser\Token`s. The most important parts of the interface can be
+summarized as follows:
 
 ```php
-$lexer = new PhpParser\Lexer(array(
-    'usedAttributes' => array(
-        'comments', 'startLine', 'endLine'
-    )
-));
+class Token {
+    /** @var int Token ID, either T_* or ord($char) for single-character tokens. */
+    public int $id;
+    /** @var string The textual content of the token. */
+    public string $text;
+    /** @var int The 1-based starting line of the token (or -1 if unknown). */
+    public int $line;
+    /** @var int The 0-based starting position of the token (or -1 if unknown). */
+    public int $pos;
+
+    /** @param int|string|(int|string)[] $kind Token ID or text (or array of them) */
+    public function is($kind): bool;
+}
 ```
 
-The attributes used in this example match the default behavior of the lexer. The following attributes are supported:
+Unlike PHP's own `PhpToken::tokenize()` output, the token array is terminated by a sentinel token with ID 0.
 
- * `comments`: Array of `PhpParser\Comment` or `PhpParser\Comment\Doc` instances, representing all comments that occurred
-   between the previous non-discarded token and the current one. Use of this attribute is required for the
-   `$node->getComments()` and `$node->getDocComment()` methods to work. The attribute is also needed if you wish the pretty
-   printer to retain comments present in the original code.
- * `startLine`: Line in which the node starts. This attribute is required for the `$node->getLine()` to work. It is also
-   required if syntax errors should contain line number information.
- * `endLine`: Line in which the node ends. Required for `$node->getEndLine()`.
- * `startTokenPos`: Offset into the token array of the first token in the node. Required for `$node->getStartTokenPos()`.
- * `endTokenPos`: Offset into the token array of the last token in the node. Required for `$node->getEndTokenPos()`.
- * `startFilePos`: Offset into the code string of the first character that is part of the node. Required for `$node->getStartFilePos()`.
- * `endFilePos`: Offset into the code string of the last character that is part of the node. Required for `$node->getEndFilePos()`.
+The lexer is normally invoked implicitly by the parser. In that case, the tokens for the last parse can be retrieved
+using `Parser::getTokens()`.
+
+Nodes in the AST produced by the parser always corresponds to some range of tokens. The parser adds a number of
+positioning attributes to allow mapping nodes back to lines, tokens or file offsets:
+
+ * `startLine`: Line in which the node starts. Used by `$node->getStartLine()`.
+ * `endLine`: Line in which the node ends. Used by `$node->getEndLine()`.
+ * `startTokenPos`: Offset into the token array of the first token in the node. Used by `$node->getStartTokenPos()`.
+ * `endTokenPos`: Offset into the token array of the last token in the node. Used by `$node->getEndTokenPos()`.
+ * `startFilePos`: Offset into the code string of the first character that is part of the node. Used by `$node->getStartFilePos()`.
+ * `endFilePos`: Offset into the code string of the last character that is part of the node. Used by `$node->getEndFilePos()`.
+
+Note that `start`/`end` here are closed rather than half-open ranges. This means that a node consisting of a single
+token will have `startTokenPos == endTokenPos` rather than `startTokenPos + 1 == endTokenPos`. This also means that a
+zero-length node will have `startTokenPos -1 == endTokenPos`.
 
 ### Using token positions
 
@@ -73,89 +111,16 @@ class MyNodeVisitor extends PhpParser\NodeVisitorAbstract {
     }
 }
 
-$lexerOptions = array(
-    'usedAttributes' => array(
-        'comments', 'startLine', 'endLine', 'startTokenPos', 'endTokenPos'
-    )
-);
 $parser = (new PhpParser\ParserFactory())->createForHostVersion($lexerOptions);
 
 $visitor = new MyNodeVisitor();
-$traverser = new PhpParser\NodeTraverser();
-$traverser->addVisitor($visitor);
+$traverser = new PhpParser\NodeTraverser($visitor);
 
 try {
     $stmts = $parser->parse($code);
-    $visitor->setTokens($lexer->getTokens());
+    $visitor->setTokens($parser->getTokens());
     $stmts = $traverser->traverse($stmts);
 } catch (PhpParser\Error $e) {
     echo 'Parse Error: ', $e->getMessage();
 }
 ```
-
-The same approach can also be used to perform specific modifications in the code, without changing the formatting in
-other places (which is the case when using the pretty printer).
-
-Lexer extension
----------------
-
-A lexer has to define the following public interface:
-
-```php
-function startLexing(string $code, ErrorHandler $errorHandler = null): void;
-function getTokens(): array;
-function handleHaltCompiler(): string;
-function getNextToken(string &$value = null, array &$startAttributes = null, array &$endAttributes = null): int;
-```
-
-The `startLexing()` method is invoked whenever the `parse()` method of the parser is called and is passed the source
-code that is to be lexed (including the opening tag). It can be used to reset state or preprocess the source code or tokens. The
-passed `ErrorHandler` should be used to report lexing errors.
-
-The `getTokens()` method returns the current array of `PhpParser\Token`s, which are compatible with the PHP 8 `PhpToken`
-class. This method is not used by the parser (which uses `getNextToken()`), but is useful in combination with the token
-position attributes.
-
-The `handleHaltCompiler()` method is called whenever a `T_HALT_COMPILER` token is encountered. It has to return the
-remaining string after the construct (not including `();`).
-
-The `getNextToken()` method returns the ID of the next token (in the sense of `Token::$id`). If no more
-tokens are available it must return `0`, which is the ID of the `EOF` token. Furthermore, the string content of the
-token should be written into the by-reference `$value` parameter (which will then be available as `$n` in the parser).
-
-### Attribute handling
-
-The other two by-ref variables `$startAttributes` and `$endAttributes` define which attributes will eventually be
-assigned to the generated nodes: The parser will take the `$startAttributes` from the first token which is part of the
-node and the `$endAttributes` from the last token that is part of the node.
-
-E.g. if the tokens `T_FUNCTION T_STRING ... '{' ... '}'` constitute a node, then the `$startAttributes` from the
-`T_FUNCTION` token will be taken and the `$endAttributes` from the `'}'` token.
-
-An application of custom attributes is storing the exact original formatting of literals: While the parser does retain
-some information about the formatting of integers (like decimal vs. hexadecimal) or strings (like used quote type), it
-does not preserve the exact original formatting (e.g. leading zeros for integers or escape sequences in strings). This
-can be remedied by storing the original value in an attribute:
-
-```php
-use PhpParser\Lexer;
-use PhpParser\Parser\Tokens;
-
-class KeepOriginalValueLexer extends Lexer // or Lexer\Emulative
-{
-    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) {
-        $tokenId = parent::getNextToken($value, $startAttributes, $endAttributes);
-
-        if ($tokenId == \T_CONSTANT_ENCAPSED_STRING   // non-interpolated string
-            || $tokenId == \T_ENCAPSED_AND_WHITESPACE // interpolated string
-            || $tokenId == \T_LNUMBER                 // integer
-            || $tokenId == \T_DNUMBER                 // floating point number
-        ) {
-            // could also use $startAttributes, doesn't really matter here
-            $endAttributes['originalValue'] = $value;
-        }
-
-        return $tokenId;
-    }
-}
-```

doc/component/Name_resolution.markdown

@@ -24,7 +24,7 @@ $stmts = $nodeTraverser->traverse($stmts);
 
 In the default configuration, the name resolver will perform three actions:
 
- * Declarations of functions, classes, interfaces, traits and global constants will have a
+ * Declarations of functions, classes, interfaces, traits, enums and global constants will have a
    `namespacedName` property added, which contains the function/class/etc name including the
    namespace prefix. For historic reasons this is a **property** rather than an attribute.
  * Names will be replaced by fully qualified resolved names, which are instances of
@@ -32,7 +32,7 @@ In the default configuration, the name resolver will perform three actions:
  * Unqualified function and constant names inside a namespace cannot be statically resolved. Inside
    a namespace `Foo`, a call to `strlen()` may either refer to the namespaced `\Foo\strlen()`, or
    the global `\strlen()`. Because PHP-Parser does not have the necessary context to decide this,
-   such names are left unresolved. Additionally a `namespacedName` **attribute** is added to the
+   such names are left unresolved. Additionally, a `namespacedName` **attribute** is added to the
    name node.
 
 The name resolver accepts an option array as the second argument, with the following default values:
@@ -47,7 +47,7 @@ $nameResolver = new PhpParser\NodeVisitor\NameResolver(null, [
 If the `preserveOriginalNames` option is enabled, then the resolved (fully qualified) name will have
 an `originalName` attribute, which contains the unresolved name.
 
-If the `replaceNodes` option is disabled, then names will no longer be resolved in-place. Instead a
+If the `replaceNodes` option is disabled, then names will no longer be resolved in-place. Instead, a
 `resolvedName` attribute will be added to each name, which contains the resolved (fully qualified)
 name. Once again, if an unqualified function or constant name cannot be resolved, then the
 `resolvedName` attribute will not be present, and instead a `namespacedName` attribute is added.
@@ -75,7 +75,7 @@ class NameContext {
 }
 ```
 
-The `$type` parameters accept on of the `Stmt\Use_::TYPE_*` constants, which represent the three
+The `$type` parameters accept one of the `Stmt\Use_::TYPE_*` constants, which represent the three
 basic symbol types in PHP (functions, constants and everything else).
 
 Next to name resolution, the `NameContext` also supports the reverse operation of finding a short

doc/component/Performance.markdown

@@ -40,5 +40,5 @@ parse multiple files.
 
 When possible, objects should be reused rather than being newly instantiated for every use. Some
 objects have expensive initialization procedures, which will be unnecessarily repeated if the object
-is not reused. (Currently two objects with particularly expensive setup are lexers and pretty
+is not reused. (Currently two objects with particularly expensive setup are parsers and pretty
 printers, though the details might change between versions of this library.)

doc/component/Pretty_printing.markdown

@@ -31,17 +31,29 @@ expression.
 Customizing the formatting
 --------------------------
 
-The pretty printer respects a number of `kind` attributes used by some notes (e.g., whether an
+The pretty printer respects a number of `kind` attributes used by some nodes (e.g., whether an
 integer should be printed as decimal, hexadecimal, etc). Additionally, it supports three options:
 
-* `phpVersion` (defaults to 7.1) allows opting into formatting that is not supported by older PHP
+* `phpVersion` (defaults to 7.4) allows opting into formatting that is not supported by older PHP
   versions.
 * `newline` (defaults to `"\n"`) can be set to `"\r\n"` in order to produce Windows newlines.
 * `shortArraySyntax` determines the used array syntax if the `kind` attribute is not set. This is
   a legacy option, and `phpVersion` should be used to control this behavior instead.
 
-However, the default pretty printer does not provide any functionality for fine-grained
-customization of code formatting.
+The behaviors controlled by `phpVersion` (defaults to PHP 7.4) are:
+
+* For PHP >= 7.0, short array syntax `[]` will be used by default. This does not affect nodes that
+  specify an explicit array syntax using the `kind` attribute.
+* For PHP >= 7.0, parentheses around `yield` expressions will only be printed when necessary.
+* For PHP >= 7.1, the short array syntax `[]` will be used for destructuring by default (instead of
+  `list()`). This does not affect nodes that specify and explicit syntax using the `kind` attribute.
+* For PHP >= 7.3, a newline is no longer forced after heredoc/nowdoc strings, as the requirement
+  for this has been removed with the introduction of flexible heredoc/nowdoc strings.
+* For PHP >= 7.3, heredoc/nowdoc strings are indented just like regular code. This was allowed with
+  the introduction of flexible heredoc/nowdoc strings.
+
+The default pretty printer does not provide functionality for fine-grained customization of code
+formatting.
 
 If you want to make minor changes to the formatting, the easiest way is to extend the pretty printer
 and override the methods responsible for the node types you are interested in.
@@ -64,29 +76,19 @@ code which has been modified or newly inserted.
 Use of the formatting-preservation functionality requires some additional preparatory steps:
 
 ```php
-use PhpParser\{Lexer, NodeTraverser, NodeVisitor, ParserFactory, PrettyPrinter};
-
-$lexerOptions = new [
-    'usedAttributes' => [
-        'comments',
-        'startLine', 'endLine',
-        'startTokenPos', 'endTokenPos',
-    ],
-];
-$parser = (new ParserFactory())->createForHostVersion($lexerOptions);
-
-$traverser = new NodeTraverser();
-$traverser->addVisitor(new NodeVisitor\CloningVisitor());
-
-$printer = new PrettyPrinter\Standard();
+use PhpParser\{NodeTraverser, NodeVisitor, ParserFactory, PrettyPrinter};
 
+$parser = (new ParserFactory())->createForHostVersion();
 $oldStmts = $parser->parse($code);
-$oldTokens = $parser->getLexer()->getTokens();
+$oldTokens = $parser->getTokens();
 
+// Run CloningVisitor before making changes to the AST.
+$traverser = new NodeTraverser(new NodeVisitor\CloningVisitor());
 $newStmts = $traverser->traverse($oldStmts);
 
 // MODIFY $newStmts HERE
 
+$printer = new PrettyPrinter\Standard();
 $newCode = $printer->printFormatPreserving($newStmts, $oldStmts, $oldTokens);
 ```
 
@@ -95,5 +97,5 @@ If you make use of the name resolution functionality, you will likely want to di
 the AST and causing spurious changes to the pretty printed code. For more information, see the
 [name resolution documentation](Name_resolution.markdown).
 
-The formatting-preservation works on a best-effort basis and may sometimes reformat more code tha
+The formatting-preservation works on a best-effort basis and may sometimes reformat more code than
 necessary. If you encounter problems while using this functionality, please open an issue.

doc/component/Walking_the_AST.markdown

@@ -21,6 +21,18 @@ $stmts = ...;
 $modifiedStmts = $traverser->traverse($stmts);
 ```
 
+Visitors can be either passed to the `NodeTraverser` constructor, or added using `addVisitor()`:
+
+```php
+$traverser = new NodeTraverser($visitor1, $visitor2, $visitor3);
+
+// Equivalent to:
+$traverser = new NodeTraverser();
+$traverser->addVisitor($visitor1);
+$traverser->addVisitor($visitor2);
+$traverser->addVisitor($visitor3);
+```
+
 Node visitors
 -------------
 
@@ -48,12 +60,11 @@ For example, if we have the following excerpt of an AST
 ```
 Expr_FuncCall(
    name: Name(
-        parts: array(
-            0: printLine
-        )
+       name: printLine
    )
    args: array(
        0: Arg(
+           name: null
            value: Scalar_String(
                value: Hello World!!!
            )
@@ -348,5 +359,6 @@ be accessed: From parents to children. However, it can often be convenient to op
 reverse direction: When working on a node, you might want to check if the parent node satisfies a
 certain property.
 
-PHP-Parser does not add parent (or sibling) references to nodes by itself, but you can easily
-emulate this with a visitor. See the [FAQ](FAQ.markdown) for more information.
+PHP-Parser does not add parent (or sibling) references to nodes by default, but you can enable them
+using the `ParentConnectingVisitor` or `NodeConnectingVisitor`. See the [FAQ](FAQ.markdown) for
+more information.

grammar/parser.template

@@ -28,68 +28,68 @@ class #(-p) extends \PhpParser\ParserAbstract
     public const %s = %n;
 #endtokenval
 
-    protected $tokenToSymbolMapSize = #(YYMAXLEX);
-    protected $actionTableSize = #(YYLAST);
-    protected $gotoTableSize = #(YYGLAST);
+    protected int $tokenToSymbolMapSize = #(YYMAXLEX);
+    protected int $actionTableSize = #(YYLAST);
+    protected int $gotoTableSize = #(YYGLAST);
 
-    protected $invalidSymbol = #(YYBADCH);
-    protected $errorSymbol = #(YYINTERRTOK);
-    protected $defaultAction = #(YYDEFAULT);
-    protected $unexpectedTokenRule = #(YYUNEXPECTED);
+    protected int $invalidSymbol = #(YYBADCH);
+    protected int $errorSymbol = #(YYINTERRTOK);
+    protected int $defaultAction = #(YYDEFAULT);
+    protected int $unexpectedTokenRule = #(YYUNEXPECTED);
 
-    protected $YY2TBLSTATE = #(YY2TBLSTATE);
-    protected $numNonLeafStates = #(YYNLSTATES);
+    protected int $YY2TBLSTATE = #(YY2TBLSTATE);
+    protected int $numNonLeafStates = #(YYNLSTATES);
 
-    protected $symbolToName = array(
+    protected array $symbolToName = array(
         #listvar terminals
     );
 
-    protected $tokenToSymbol = array(
+    protected array $tokenToSymbol = array(
         #listvar yytranslate
     );
 
-    protected $action = array(
+    protected array $action = array(
         #listvar yyaction
     );
 
-    protected $actionCheck = array(
+    protected array $actionCheck = array(
         #listvar yycheck
     );
 
-    protected $actionBase = array(
+    protected array $actionBase = array(
         #listvar yybase
     );
 
-    protected $actionDefault = array(
+    protected array $actionDefault = array(
         #listvar yydefault
     );
 
-    protected $goto = array(
+    protected array $goto = array(
         #listvar yygoto
     );
 
-    protected $gotoCheck = array(
+    protected array $gotoCheck = array(
         #listvar yygcheck
     );
 
-    protected $gotoBase = array(
+    protected array $gotoBase = array(
         #listvar yygbase
     );
 
-    protected $gotoDefault = array(
+    protected array $gotoDefault = array(
         #listvar yygdefault
     );
 
-    protected $ruleToNonTerminal = array(
+    protected array $ruleToNonTerminal = array(
         #listvar yylhs
     );
 
-    protected $ruleToLength = array(
+    protected array $ruleToLength = array(
         #listvar yylen
     );
 #if -t
 
-    protected $productions = array(
+    protected array $productions = array(
         #production-strings;
     );
 #endif
@@ -101,9 +101,7 @@ class #(-p) extends \PhpParser\ParserAbstract
                 %b
             },
 #noact
-            %n => function ($stackPos) {
-                $this->semValue = $this->semStack[$stackPos];
-            },
+            %n => null,
 #endreduce
         ];
     }

grammar/php.y

@@ -134,7 +134,7 @@ top_statement_list_ex:
 
 top_statement_list:
       top_statement_list_ex
-          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
+          { makeZeroLengthNop($nop);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -185,7 +185,7 @@ namespace_name:
 ;
 
 legacy_namespace_name:
-      namespace_name                                        { $$ = $1; }
+      namespace_name
     | T_NAME_FULLY_QUALIFIED                                { $$ = Name[substr($1, 1)]; }
 ;
 
@@ -229,15 +229,15 @@ attributes:
 
 optional_attributes:
       /* empty */                                           { $$ = []; }
-    | attributes                                            { $$ = $1; }
+    | attributes
 ;
 
 top_statement:
-      statement                                             { $$ = $1; }
-    | function_declaration_statement                        { $$ = $1; }
-    | class_declaration_statement                           { $$ = $1; }
+      statement
+    | function_declaration_statement
+    | class_declaration_statement
     | T_HALT_COMPILER '(' ')' ';'
-          { $$ = Stmt\HaltCompiler[$this->lexer->handleHaltCompiler()]; }
+          { $$ = Stmt\HaltCompiler[$this->handleHaltCompiler()]; }
     | T_NAMESPACE namespace_declaration_name semi
           { $$ = Stmt\Namespace_[$2, null];
             $$->setAttribute('kind', Stmt\Namespace_::KIND_SEMICOLON);
@@ -252,7 +252,7 @@ top_statement:
             $this->checkNamespace($$); }
     | T_USE use_declarations semi                           { $$ = Stmt\Use_[$2, Stmt\Use_::TYPE_NORMAL]; }
     | T_USE use_type use_declarations semi                  { $$ = Stmt\Use_[$3, $2]; }
-    | group_use_declaration semi                            { $$ = $1; }
+    | group_use_declaration semi
     | T_CONST constant_declaration_list semi                { $$ = Stmt\Const_[$2]; }
 ;
 
@@ -269,7 +269,7 @@ group_use_declaration:
 ;
 
 unprefixed_use_declarations:
-      non_empty_unprefixed_use_declarations optional_comma  { $$ = $1; }
+      non_empty_unprefixed_use_declarations optional_comma
 ;
 
 non_empty_unprefixed_use_declarations:
@@ -279,7 +279,7 @@ non_empty_unprefixed_use_declarations:
 ;
 
 use_declarations:
-      non_empty_use_declarations no_comma                   { $$ = $1; }
+      non_empty_use_declarations no_comma
 ;
 
 non_empty_use_declarations:
@@ -288,7 +288,7 @@ non_empty_use_declarations:
 ;
 
 inline_use_declarations:
-      non_empty_inline_use_declarations optional_comma      { $$ = $1; }
+      non_empty_inline_use_declarations optional_comma
 ;
 
 non_empty_inline_use_declarations:
@@ -317,7 +317,7 @@ inline_use_declaration:
 ;
 
 constant_declaration_list:
-      non_empty_constant_declaration_list no_comma          { $$ = $1; }
+      non_empty_constant_declaration_list no_comma
 ;
 
 non_empty_constant_declaration_list:
@@ -331,7 +331,7 @@ constant_declaration:
 ;
 
 class_const_list:
-      non_empty_class_const_list no_comma                   { $$ = $1; }
+      non_empty_class_const_list no_comma
 ;
 
 non_empty_class_const_list:
@@ -353,34 +353,26 @@ inner_statement_list_ex:
 
 inner_statement_list:
       inner_statement_list_ex
-          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
+          { makeZeroLengthNop($nop);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
 inner_statement:
-      statement                                             { $$ = $1; }
-    | function_declaration_statement                        { $$ = $1; }
-    | class_declaration_statement                           { $$ = $1; }
+      statement
+    | function_declaration_statement
+    | class_declaration_statement
     | T_HALT_COMPILER
           { throw new Error('__HALT_COMPILER() can only be used from the outermost scope', attributes()); }
 ;
 
 non_empty_statement:
-      '{' inner_statement_list '}'
-    {
-        if ($2) {
-            $$ = $2; prependLeadingComments($$);
-        } else {
-            makeNop($$, $this->startAttributeStack[#1], $this->endAttributes);
-            if (null === $$) { $$ = array(); }
-        }
-    }
-    | T_IF '(' expr ')' statement elseif_list else_single
-          { $$ = Stmt\If_[$3, ['stmts' => toArray($5), 'elseifs' => $6, 'else' => $7]]; }
+      '{' inner_statement_list '}'                          { $$ = Stmt\Block[$2]; }
+    | T_IF '(' expr ')' blocklike_statement elseif_list else_single
+          { $$ = Stmt\If_[$3, ['stmts' => $5, 'elseifs' => $6, 'else' => $7]]; }
     | T_IF '(' expr ')' ':' inner_statement_list new_elseif_list new_else_single T_ENDIF ';'
           { $$ = Stmt\If_[$3, ['stmts' => $6, 'elseifs' => $7, 'else' => $8]]; }
     | T_WHILE '(' expr ')' while_statement                  { $$ = Stmt\While_[$3, $5]; }
-    | T_DO statement T_WHILE '(' expr ')' ';'               { $$ = Stmt\Do_   [$5, toArray($2)]; }
+    | T_DO blocklike_statement T_WHILE '(' expr ')' ';'     { $$ = Stmt\Do_   [$5, $2]; }
     | T_FOR '(' for_expr ';'  for_expr ';' for_expr ')' for_statement
           { $$ = Stmt\For_[['init' => $3, 'cond' => $5, 'loop' => $7, 'stmts' => $9]]; }
     | T_SWITCH '(' expr ')' switch_case_list                { $$ = Stmt\Switch_[$3, $5]; }
@@ -390,17 +382,11 @@ non_empty_statement:
     | T_GLOBAL global_var_list semi                         { $$ = Stmt\Global_[$2]; }
     | T_STATIC static_var_list semi                         { $$ = Stmt\Static_[$2]; }
     | T_ECHO expr_list_forbid_comma semi                    { $$ = Stmt\Echo_[$2]; }
-    | T_INLINE_HTML                                         { $$ = Stmt\InlineHTML[$1]; }
-    | expr semi {
-        $e = $1;
-        if ($e instanceof Expr\Throw_) {
-            // For backwards-compatibility reasons, convert throw in statement position into
-            // Stmt\Throw_ rather than Stmt\Expression(Expr\Throw_).
-            $$ = Stmt\Throw_[$e->expr];
-        } else {
-            $$ = Stmt\Expression[$e];
-        }
+    | T_INLINE_HTML {
+        $$ = Stmt\InlineHTML[$1];
+        $$->setAttribute('hasLeadingNewline', $this->inlineHtmlHasLeadingNewline(#1));
     }
+    | expr semi                                             { $$ = Stmt\Expression[$1]; }
     | T_UNSET '(' variables_list ')' semi                   { $$ = Stmt\Unset_[$3]; }
     | T_FOREACH '(' expr T_AS foreach_variable ')' foreach_statement
           { $$ = Stmt\Foreach_[$3, $5[0], ['keyVar' => null, 'byRef' => $5[1], 'stmts' => $7]]; }
@@ -413,14 +399,16 @@ non_empty_statement:
           { $$ = Stmt\TryCatch[$3, $5, $6]; $this->checkTryCatch($$); }
     | T_GOTO identifier_not_reserved semi                   { $$ = Stmt\Goto_[$2]; }
     | identifier_not_reserved ':'                           { $$ = Stmt\Label[$1]; }
-    | error                                                 { $$ = array(); /* means: no statement */ }
+    | error                                                 { $$ = null; /* means: no statement */ }
 ;
 
 statement:
-      non_empty_statement                                   { $$ = $1; }
-    | ';'
-          { makeNop($$, $this->startAttributeStack[#1], $this->endAttributes);
-            if ($$ === null) $$ = array(); /* means: no statement */ }
+      non_empty_statement
+    | ';'                                                   { makeNop($$); }
+;
+
+blocklike_statement:
+     statement                                              { toBlock($1); }
 ;
 
 catches:
@@ -444,7 +432,7 @@ optional_finally:
 ;
 
 variables_list:
-      non_empty_variables_list optional_comma               { $$ = $1; }
+      non_empty_variables_list optional_comma
 ;
 
 non_empty_variables_list:
@@ -473,7 +461,7 @@ block_or_error:
 ;
 
 identifier_maybe_readonly:
-      identifier_not_reserved                               { $$ = $1; }
+      identifier_not_reserved
     | T_READONLY                                            { $$ = Node\Identifier[$1]; }
 ;
 
@@ -512,11 +500,11 @@ enum_case_expr:
 
 class_entry_type:
       T_CLASS                                               { $$ = 0; }
-    | class_modifiers T_CLASS                               { $$ = $1; }
+    | class_modifiers T_CLASS
 ;
 
 class_modifiers:
-      class_modifier                                        { $$ = $1; }
+      class_modifier
     | class_modifiers class_modifier                        { $this->checkClassModifier($1, $2, #2); $$ = $1 | $2; }
 ;
 
@@ -542,7 +530,7 @@ implements_list:
 ;
 
 class_name_list:
-      non_empty_class_name_list no_comma                    { $$ = $1; }
+      non_empty_class_name_list no_comma
 ;
 
 non_empty_class_name_list:
@@ -551,23 +539,23 @@ non_empty_class_name_list:
 ;
 
 for_statement:
-      statement                                             { $$ = toArray($1); }
+      blocklike_statement
     | ':' inner_statement_list T_ENDFOR ';'                 { $$ = $2; }
 ;
 
 foreach_statement:
-      statement                                             { $$ = toArray($1); }
+      blocklike_statement
     | ':' inner_statement_list T_ENDFOREACH ';'             { $$ = $2; }
 ;
 
 declare_statement:
-      non_empty_statement                                   { $$ = toArray($1); }
+      non_empty_statement                                   { toBlock($1); }
     | ';'                                                   { $$ = null; }
     | ':' inner_statement_list T_ENDDECLARE ';'             { $$ = $2; }
 ;
 
 declare_list:
-      non_empty_declare_list no_comma                       { $$ = $1; }
+      non_empty_declare_list no_comma
 ;
 
 non_empty_declare_list:
@@ -607,7 +595,7 @@ match:
 
 match_arm_list:
       /* empty */                                           { $$ = []; }
-    | non_empty_match_arm_list optional_comma               { $$ = $1; }
+    | non_empty_match_arm_list optional_comma
 ;
 
 non_empty_match_arm_list:
@@ -621,7 +609,7 @@ match_arm:
 ;
 
 while_statement:
-      statement                                             { $$ = toArray($1); }
+      blocklike_statement                                   { $$ = $1; }
     | ':' inner_statement_list T_ENDWHILE ';'               { $$ = $2; }
 ;
 
@@ -631,7 +619,7 @@ elseif_list:
 ;
 
 elseif:
-      T_ELSEIF '(' expr ')' statement                       { $$ = Stmt\ElseIf_[$3, toArray($5)]; }
+      T_ELSEIF '(' expr ')' blocklike_statement             { $$ = Stmt\ElseIf_[$3, $5]; }
 ;
 
 new_elseif_list:
@@ -646,7 +634,7 @@ new_elseif:
 
 else_single:
       /* empty */                                           { $$ = null; }
-    | T_ELSE statement                                      { $$ = Stmt\Else_[toArray($2)]; }
+    | T_ELSE blocklike_statement                            { $$ = Stmt\Else_[$2]; }
 ;
 
 new_else_single:
@@ -664,7 +652,7 @@ foreach_variable:
 ;
 
 parameter_list:
-      non_empty_parameter_list optional_comma               { $$ = $1; }
+      non_empty_parameter_list optional_comma
     | /* empty */                                           { $$ = array(); }
 ;
 
@@ -701,14 +689,14 @@ parameter:
 ;
 
 type_expr:
-      type                                                  { $$ = $1; }
+      type
     | '?' type                                              { $$ = Node\NullableType[$2]; }
     | union_type                                            { $$ = Node\UnionType[$1]; }
-    | intersection_type                                     { $$ = $1; }
+    | intersection_type
 ;
 
 type:
-      type_without_static                                   { $$ = $1; }
+      type_without_static
     | T_STATIC                                              { $$ = Node\Name['static']; }
 ;
 
@@ -719,7 +707,7 @@ type_without_static:
 ;
 
 union_type_element:
-                type { $$ = $1; }
+      type
     | '(' intersection_type ')' { $$ = $2; }
 ;
 
@@ -729,7 +717,7 @@ union_type:
 ;
 
 union_type_without_static_element:
-                type_without_static { $$ = $1; }
+                type_without_static
         |        '(' intersection_type_without_static ')' { $$ = $2; }
 ;
 
@@ -760,15 +748,15 @@ intersection_type_without_static:
 ;
 
 type_expr_without_static:
-      type_without_static                                   { $$ = $1; }
+      type_without_static
     | '?' type_without_static                               { $$ = Node\NullableType[$2]; }
     | union_type_without_static                             { $$ = Node\UnionType[$1]; }
-    | intersection_type_without_static                      { $$ = $1; }
+    | intersection_type_without_static
 ;
 
 optional_type_without_static:
       /* empty */                                           { $$ = null; }
-    | type_expr_without_static                              { $$ = $1; }
+    | type_expr_without_static
 ;
 
 optional_return_type:
@@ -801,7 +789,7 @@ argument:
 ;
 
 global_var_list:
-      non_empty_global_var_list no_comma                    { $$ = $1; }
+      non_empty_global_var_list no_comma
 ;
 
 non_empty_global_var_list:
@@ -810,11 +798,11 @@ non_empty_global_var_list:
 ;
 
 global_var:
-      simple_variable                                       { $$ = $1; }
+      simple_variable
 ;
 
 static_var_list:
-      non_empty_static_var_list no_comma                    { $$ = $1; }
+      non_empty_static_var_list no_comma
 ;
 
 non_empty_static_var_list:
@@ -828,13 +816,13 @@ static_var:
 ;
 
 class_statement_list_ex:
-      class_statement_list_ex class_statement               { if ($2 !== null) { push($1, $2); } }
+      class_statement_list_ex class_statement               { if ($2 !== null) { push($1, $2); } else { $$ = $1; } }
     | /* empty */                                           { init(); }
 ;
 
 class_statement_list:
       class_statement_list_ex
-          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
+          { makeZeroLengthNop($nop);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -885,27 +873,27 @@ trait_method_reference_fully_qualified:
       name T_PAAMAYIM_NEKUDOTAYIM identifier_maybe_reserved { $$ = array($1, $3); }
 ;
 trait_method_reference:
-      trait_method_reference_fully_qualified                { $$ = $1; }
+      trait_method_reference_fully_qualified
     | identifier_maybe_reserved                             { $$ = array(null, $1); }
 ;
 
 method_body:
       ';' /* abstract method */                             { $$ = null; }
-    | block_or_error                                        { $$ = $1; }
+    | block_or_error
 ;
 
 variable_modifiers:
-      non_empty_member_modifiers                            { $$ = $1; }
+      non_empty_member_modifiers
     | T_VAR                                                 { $$ = 0; }
 ;
 
 method_modifiers:
       /* empty */                                           { $$ = 0; }
-    | non_empty_member_modifiers                            { $$ = $1; }
+    | non_empty_member_modifiers
 ;
 
 non_empty_member_modifiers:
-      member_modifier                                       { $$ = $1; }
+      member_modifier
     | non_empty_member_modifiers member_modifier            { $this->checkModifier($1, $2, #2); $$ = $1 | $2; }
 ;
 
@@ -920,7 +908,7 @@ member_modifier:
 ;
 
 property_declaration_list:
-      non_empty_property_declaration_list no_comma          { $$ = $1; }
+      non_empty_property_declaration_list no_comma
 ;
 
 non_empty_property_declaration_list:
@@ -939,11 +927,11 @@ property_declaration:
 ;
 
 expr_list_forbid_comma:
-      non_empty_expr_list no_comma                          { $$ = $1; }
+      non_empty_expr_list no_comma
 ;
 
 expr_list_allow_comma:
-      non_empty_expr_list optional_comma                    { $$ = $1; }
+      non_empty_expr_list optional_comma
 ;
 
 non_empty_expr_list:
@@ -953,11 +941,11 @@ non_empty_expr_list:
 
 for_expr:
       /* empty */                                           { $$ = array(); }
-    | expr_list_forbid_comma                                { $$ = $1; }
+    | expr_list_forbid_comma
 ;
 
 expr:
-      variable                                              { $$ = $1; }
+      variable
     | list_expr '=' expr                                    { $$ = Expr\Assign[$1, $3]; }
     | array_short_syntax '=' expr
           { $$ = Expr\Assign[$this->fixupArrayDestructuring($1), $3]; }
@@ -969,8 +957,8 @@ expr:
                 $this->emitError(new Error('Cannot assign new by reference', attributes()));
             }
           }
-    | new_expr                                              { $$ = $1; }
-    | match                                                 { $$ = $1; }
+    | new_expr
+    | match
     | T_CLONE expr                                          { $$ = Expr\Clone_[$2]; }
     | variable T_PLUS_EQUAL expr                            { $$ = Expr\AssignOp\Plus      [$1, $3]; }
     | variable T_MINUS_EQUAL expr                           { $$ = Expr\AssignOp\Minus     [$1, $3]; }
@@ -1047,7 +1035,7 @@ expr:
             $attrs['kind'] = strtolower($1) === 'exit' ? Expr\Exit_::KIND_EXIT : Expr\Exit_::KIND_DIE;
             $$ = new Expr\Exit_($2, $attrs); }
     | '@' expr                                              { $$ = Expr\ErrorSuppress[$2]; }
-    | scalar                                                { $$ = $1; }
+    | scalar
     | '`' backticks_expr '`'                                { $$ = Expr\ShellExec[$2]; }
     | T_PRINT expr                                          { $$ = Expr\Print_[$2]; }
     | T_YIELD                                               { $$ = Expr\Yield_[null, null]; }
@@ -1093,7 +1081,7 @@ lexical_vars:
 ;
 
 lexical_var_list:
-      non_empty_lexical_var_list optional_comma             { $$ = $1; }
+      non_empty_lexical_var_list optional_comma
 ;
 
 non_empty_lexical_var_list:
@@ -1119,7 +1107,7 @@ function_call:
 
 class_name:
       T_STATIC                                              { $$ = Name[$1]; }
-    | name                                                  { $$ = $1; }
+    | name
 ;
 
 name:
@@ -1130,15 +1118,15 @@ name:
 ;
 
 class_name_reference:
-      class_name                                            { $$ = $1; }
-    | new_variable                                          { $$ = $1; }
+      class_name
+    | new_variable
     | '(' expr ')'                                          { $$ = $2; }
     | error                                                 { $$ = Expr\Error[]; $this->errorState = 2; }
 ;
 
 class_name_or_var:
-      class_name                                            { $$ = $1; }
-    | fully_dereferencable                                  { $$ = $1; }
+      class_name
+    | fully_dereferencable
 ;
 
 exit_expr:
@@ -1148,14 +1136,14 @@ exit_expr:
 
 backticks_expr:
       /* empty */                                           { $$ = array(); }
-    | T_ENCAPSED_AND_WHITESPACE
-          { $$ = array(Node\InterpolatedStringPart[Scalar\String_::parseEscapeSequences($1, '`')]); }
-    | encaps_list                                           { parseEncapsed($1, '`', true); $$ = $1; }
+    | encaps_string_part
+          { $$ = array($1); parseEncapsed($$, '`', $this->phpVersion->supportsUnicodeEscapes()); }
+    | encaps_list                                           { parseEncapsed($1, '`', $this->phpVersion->supportsUnicodeEscapes()); $$ = $1; }
 ;
 
 ctor_arguments:
       /* empty */                                           { $$ = array(); }
-    | argument_list                                         { $$ = $1; }
+    | argument_list
 ;
 
 constant:
@@ -1193,19 +1181,20 @@ dereferencable_scalar:
             $$ = new Expr\Array_($3, $attrs);
             $this->createdArrays->attach($$); }
     | array_short_syntax                                    { $$ = $1; $this->createdArrays->attach($$); }
-    | T_CONSTANT_ENCAPSED_STRING                            { $$ = Scalar\String_::fromString($1, attributes()); }
+    | T_CONSTANT_ENCAPSED_STRING
+          { $$ = Scalar\String_::fromString($1, attributes(), $this->phpVersion->supportsUnicodeEscapes()); }
     | '"' encaps_list '"'
           { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
-            parseEncapsed($2, '"', true); $$ = new Scalar\InterpolatedString($2, $attrs); }
+            parseEncapsed($2, '"', $this->phpVersion->supportsUnicodeEscapes()); $$ = new Scalar\InterpolatedString($2, $attrs); }
 ;
 
 scalar:
       T_LNUMBER
           { $$ = $this->parseLNumber($1, attributes(), $this->phpVersion->allowsInvalidOctals()); }
     | T_DNUMBER                                             { $$ = Scalar\Float_::fromString($1, attributes()); }
-    | dereferencable_scalar                                 { $$ = $1; }
-    | constant                                              { $$ = $1; }
-    | class_constant                                        { $$ = $1; }
+    | dereferencable_scalar
+    | constant
+    | class_constant
     | T_START_HEREDOC T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC
           { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), true); }
     | T_START_HEREDOC T_END_HEREDOC
@@ -1216,32 +1205,32 @@ scalar:
 
 optional_expr:
       /* empty */                                           { $$ = null; }
-    | expr                                                  { $$ = $1; }
+    | expr
 ;
 
 fully_dereferencable:
-      variable                                              { $$ = $1; }
+      variable
     | '(' expr ')'                                          { $$ = $2; }
-    | dereferencable_scalar                                 { $$ = $1; }
-    | class_constant                                        { $$ = $1; }
+    | dereferencable_scalar
+    | class_constant
 ;
 
 array_object_dereferencable:
-      fully_dereferencable                                  { $$ = $1; }
-    | constant                                              { $$ = $1; }
+      fully_dereferencable
+    | constant
 ;
 
 callable_expr:
-      callable_variable                                     { $$ = $1; }
+      callable_variable
     | '(' expr ')'                                          { $$ = $2; }
-    | dereferencable_scalar                                 { $$ = $1; }
+    | dereferencable_scalar
 ;
 
 callable_variable:
-      simple_variable                                       { $$ = $1; }
+      simple_variable
     | array_object_dereferencable '[' optional_expr ']'     { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | array_object_dereferencable '{' expr '}'              { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | function_call                                         { $$ = $1; }
+    | function_call
     | array_object_dereferencable T_OBJECT_OPERATOR property_name argument_list
           { $$ = Expr\MethodCall[$1, $3, $4]; }
     | array_object_dereferencable T_NULLSAFE_OBJECT_OPERATOR property_name argument_list
@@ -1250,12 +1239,12 @@ callable_variable:
 
 optional_plain_variable:
       /* empty */                                           { $$ = null; }
-    | plain_variable                                        { $$ = $1; }
+    | plain_variable
 ;
 
 variable:
-      callable_variable                                     { $$ = $1; }
-    | static_member                                         { $$ = $1; }
+      callable_variable
+    | static_member
     | array_object_dereferencable T_OBJECT_OPERATOR property_name
           { $$ = Expr\PropertyFetch[$1, $3]; }
     | array_object_dereferencable T_NULLSAFE_OBJECT_OPERATOR property_name
@@ -1263,7 +1252,7 @@ variable:
 ;
 
 simple_variable:
-      plain_variable                                        { $$ = $1; }
+      plain_variable
     | '$' '{' expr '}'                                      { $$ = Expr\Variable[$3]; }
     | '$' simple_variable                                   { $$ = Expr\Variable[$2]; }
     | '$' error                                             { $$ = Expr\Variable[Expr\Error[]]; $this->errorState = 2; }
@@ -1280,7 +1269,7 @@ static_member:
 ;
 
 new_variable:
-      simple_variable                                       { $$ = $1; }
+      simple_variable
     | new_variable '[' optional_expr ']'                    { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | new_variable '{' expr '}'                             { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | new_variable T_OBJECT_OPERATOR property_name          { $$ = Expr\PropertyFetch[$1, $3]; }
@@ -1292,15 +1281,15 @@ new_variable:
 ;
 
 member_name:
-      identifier_maybe_reserved                             { $$ = $1; }
+      identifier_maybe_reserved
     | '{' expr '}'                                          { $$ = $2; }
-    | simple_variable                                       { $$ = $1; }
+    | simple_variable
 ;
 
 property_name:
-      identifier_not_reserved                               { $$ = $1; }
+      identifier_not_reserved
     | '{' expr '}'                                          { $$ = $2; }
-    | simple_variable                                       { $$ = $1; }
+    | simple_variable
     | error                                                 { $$ = Expr\Error[]; $this->errorState = 2; }
 ;
 
@@ -1337,7 +1326,7 @@ array_pair:
     | /* empty */
         { /* Create an Error node now to remember the position. We'll later either report an error,
              or convert this into a null element, depending on whether this is a creation or destructuring context. */
-          $attrs = $this->createEmptyElemAttributes($this->lookaheadStartAttributes);
+          $attrs = $this->createEmptyElemAttributes($this->tokenPos);
           $$ = new Node\ArrayItem(new Expr\Error($attrs), null, false, $attrs); }
 ;
 
@@ -1349,7 +1338,8 @@ encaps_list:
 ;
 
 encaps_string_part:
-      T_ENCAPSED_AND_WHITESPACE                             { $$ = Node\InterpolatedStringPart[$1]; }
+      T_ENCAPSED_AND_WHITESPACE
+          { $attrs = attributes(); $attrs['rawValue'] = $1; $$ = new Node\InterpolatedStringPart($1, $attrs); }
 ;
 
 encaps_str_varname:
@@ -1357,7 +1347,7 @@ encaps_str_varname:
 ;
 
 encaps_var:
-      plain_variable                                        { $$ = $1; }
+      plain_variable
     | plain_variable '[' encaps_var_offset ']'              { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | plain_variable T_OBJECT_OPERATOR identifier_not_reserved
           { $$ = Expr\PropertyFetch[$1, $3]; }
@@ -1374,7 +1364,7 @@ encaps_var_offset:
       T_STRING                                              { $$ = Scalar\String_[$1]; }
     | T_NUM_STRING                                          { $$ = $this->parseNumString($1, attributes()); }
     | '-' T_NUM_STRING                                      { $$ = $this->parseNumString('-' . $2, attributes()); }
-    | plain_variable                                        { $$ = $1; }
+    | plain_variable
 ;
 
 %%

grammar/phpyLang.php

@@ -65,13 +65,13 @@ function resolveMacros($code) {
 
             if ('attributes' === $name) {
                 assertArgs(0, $args, $name);
-                return '$this->startAttributeStack[#1] + $this->endAttributes';
+                return '$this->getAttributes($this->tokenStartStack[#1], $this->tokenEndStack[$stackPos])';
             }
 
             if ('stackAttributes' === $name) {
                 assertArgs(1, $args, $name);
-                return '$this->startAttributeStack[' . $args[0] . ']'
-                       . ' + $this->endAttributeStack[' . $args[0] . ']';
+                return '$this->getAttributes($this->tokenStartStack[' . $args[0] . '], '
+                       . ' $this->tokenEndStack[' . $args[0] . '])';
             }
 
             if ('init' === $name) {
@@ -87,14 +87,15 @@ function resolveMacros($code) {
             if ('pushNormalizing' === $name) {
                 assertArgs(2, $args, $name);
 
-                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
-                       . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
+                return 'if (' . $args[1] . ' !== null) { ' . $args[0] . '[] = ' . $args[1] . '; } $$ = ' . $args[0] . ';';
             }
 
-            if ('toArray' == $name) {
+            if ('toBlock' == $name) {
                 assertArgs(1, $args, $name);
 
-                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
+                return 'if (' . $args[0] . ' instanceof Stmt\Block) { $$ = ' . $args[0] . '->stmts; } '
+                     . 'else if (' . $args[0] . ' === null) { $$ = []; } '
+                     . 'else { $$ = [' . $args[0] . ']; }';
             }
 
             if ('parseVar' === $name) {
@@ -111,30 +112,15 @@ function resolveMacros($code) {
             }
 
             if ('makeNop' === $name) {
-                assertArgs(3, $args, $name);
+                assertArgs(1, $args, $name);
 
-                return '$startAttributes = ' . $args[1] . ';'
-                       . ' if (isset($startAttributes[\'comments\']))'
-                       . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
-                       . ' else { ' . $args[0] . ' = null; }';
+                return $args[0] . ' = $this->maybeCreateNop($this->tokenStartStack[#1], $this->tokenEndStack[$stackPos])';
             }
 
             if ('makeZeroLengthNop' == $name) {
-                assertArgs(2, $args, $name);
-
-                return '$startAttributes = ' . $args[1] . ';'
-                       . ' if (isset($startAttributes[\'comments\']))'
-                       . ' { ' . $args[0] . ' = new Stmt\Nop($this->createCommentNopAttributes($startAttributes[\'comments\'])); }'
-                       . ' else { ' . $args[0] . ' = null; }';
-            }
-
-            if ('prependLeadingComments' === $name) {
                 assertArgs(1, $args, $name);
 
-                return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
-                       . 'if (!empty($attrs[\'comments\'])) {'
-                       . '$stmts[0]->setAttribute(\'comments\', '
-                       . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
+                return $args[0] . ' = $this->maybeCreateZeroLengthNop($this->tokenPos);';
             }
 
             return $matches[0];

lib/PhpParser/Builder/ClassConst.php

@@ -13,17 +13,16 @@ use PhpParser\Node\Identifier;
 use PhpParser\Node\Stmt;
 
 class ClassConst implements PhpParser\Builder {
-    /** @var int */
-    protected $flags = 0;
+    protected int $flags = 0;
     /** @var array<string, mixed> */
-    protected $attributes = [];
+    protected array $attributes = [];
     /** @var list<Const_> */
-    protected $constants = [];
+    protected array $constants = [];
 
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
-    /** @var Identifier|Node\Name|Node\ComplexType */
-    protected $type;
+    protected array $attributeGroups = [];
+    /** @var Identifier|Node\Name|Node\ComplexType|null */
+    protected ?Node $type = null;
 
     /**
      * Creates a class constant builder

lib/PhpParser/Builder/Class_.php

@@ -10,26 +10,21 @@ use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
 class Class_ extends Declaration {
-    /** @var string */
-    protected $name;
-
-    /** @var Name|null */
-    protected $extends = null;
+    protected string $name;
+    protected ?Name $extends = null;
     /** @var list<Name> */
-    protected $implements = [];
-    /** @var int */
-    protected $flags = 0;
-
+    protected array $implements = [];
+    protected int $flags = 0;
     /** @var list<Stmt\TraitUse> */
-    protected $uses = [];
+    protected array $uses = [];
     /** @var list<Stmt\ClassConst> */
-    protected $constants = [];
+    protected array $constants = [];
     /** @var list<Stmt\Property> */
-    protected $properties = [];
+    protected array $properties = [];
     /** @var list<Stmt\ClassMethod> */
-    protected $methods = [];
+    protected array $methods = [];
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates a class builder.

lib/PhpParser/Builder/Declaration.php

@@ -7,7 +7,7 @@ use PhpParser\BuilderHelpers;
 
 abstract class Declaration implements PhpParser\Builder {
     /** @var array<string, mixed> */
-    protected $attributes = [];
+    protected array $attributes = [];
 
     /**
      * Adds a statement.

lib/PhpParser/Builder/EnumCase.php

@@ -14,12 +14,12 @@ class EnumCase implements PhpParser\Builder {
     /** @var Identifier|string */
     protected $name;
     /** @var ?Node\Expr */
-    protected $value = null;
+    protected ?Node\Expr $value = null;
     /** @var array<string, mixed> */
-    protected $attributes = [];
+    protected array $attributes = [];
 
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates an enum case builder.

lib/PhpParser/Builder/Enum_.php

@@ -10,22 +10,20 @@ use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
 class Enum_ extends Declaration {
-    /** @var string */
-    protected $name;
-    /** @var Identifier|null */
-    protected $scalarType = null;
+    protected string $name;
+    protected ?Identifier $scalarType = null;
     /** @var list<Name> */
-    protected $implements = [];
+    protected array $implements = [];
     /** @var list<Stmt\TraitUse> */
-    protected $uses = [];
+    protected array $uses = [];
     /** @var list<Stmt\EnumCase> */
-    protected $enumCases = [];
+    protected array $enumCases = [];
     /** @var list<Stmt\ClassConst> */
-    protected $constants = [];
+    protected array $constants = [];
     /** @var list<Stmt\ClassMethod> */
-    protected $methods = [];
+    protected array $methods = [];
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates an enum builder.

lib/PhpParser/Builder/FunctionLike.php

@@ -6,13 +6,12 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 
 abstract class FunctionLike extends Declaration {
-    /** @var bool */
-    protected $returnByRef = false;
+    protected bool $returnByRef = false;
     /** @var Node\Param[] */
-    protected $params = [];
+    protected array $params = [];
 
     /** @var Node\Identifier|Node\Name|Node\ComplexType|null */
-    protected $returnType = null;
+    protected ?Node $returnType = null;
 
     /**
      * Make the function return by reference.

lib/PhpParser/Builder/Function_.php

@@ -8,13 +8,12 @@ use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
 class Function_ extends FunctionLike {
-    /** @var string */
-    protected $name;
+    protected string $name;
     /** @var list<Stmt> */
-    protected $stmts = [];
+    protected array $stmts = [];
 
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates a function builder.

lib/PhpParser/Builder/Interface_.php

@@ -9,16 +9,15 @@ use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
 class Interface_ extends Declaration {
-    /** @var string */
-    protected $name;
+    protected string $name;
     /** @var list<Name> */
-    protected $extends = [];
+    protected array $extends = [];
     /** @var list<Stmt\ClassConst> */
-    protected $constants = [];
+    protected array $constants = [];
     /** @var list<Stmt\ClassMethod> */
-    protected $methods = [];
+    protected array $methods = [];
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates an interface builder.

lib/PhpParser/Builder/Method.php

@@ -9,16 +9,15 @@ use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
 class Method extends FunctionLike {
-    /** @var string */
-    protected $name;
-    /** @var int */
-    protected $flags = 0;
+    protected string $name;
+
+    protected int $flags = 0;
 
     /** @var list<Stmt>|null */
-    protected $stmts = [];
+    protected ?array $stmts = [];
 
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates a method builder.

lib/PhpParser/Builder/Namespace_.php

@@ -8,10 +8,9 @@ use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
 class Namespace_ extends Declaration {
-    /** @var Node\Name|null */
-    private $name;
+    private ?Node\Name $name;
     /** @var Stmt[] */
-    private $stmts = [];
+    private array $stmts = [];
 
     /**
      * Creates a namespace builder.

lib/PhpParser/Builder/Param.php

@@ -8,20 +8,15 @@ use PhpParser\Modifiers;
 use PhpParser\Node;
 
 class Param implements PhpParser\Builder {
-    /** @var string */
-    protected $name;
-    /** @var Node\Expr|null */
-    protected $default = null;
+    protected string $name;
+    protected ?Node\Expr $default = null;
     /** @var Node\Identifier|Node\Name|Node\ComplexType|null */
-    protected $type = null;
-    /** @var bool */
-    protected $byRef = false;
-    /** @var int */
-    protected $flags = 0;
-    /** @var bool */
-    protected $variadic = false;
+    protected ?Node $type = null;
+    protected bool $byRef = false;
+    protected int $flags = 0;
+    protected bool $variadic = false;
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates a parameter builder.

lib/PhpParser/Builder/Property.php

@@ -12,18 +12,17 @@ use PhpParser\Node\Stmt;
 use PhpParser\Node\ComplexType;
 
 class Property implements PhpParser\Builder {
-    /** @var string */
-    protected $name;
-    /** @var int */
-    protected $flags = 0;
-    /** @var Node\Expr|null */
-    protected $default = null;
+    protected string $name;
+
+    protected int $flags = 0;
+
+    protected ?Node\Expr $default = null;
     /** @var array<string, mixed> */
-    protected $attributes = [];
+    protected array $attributes = [];
     /** @var null|Identifier|Name|ComplexType */
-    protected $type;
+    protected ?Node $type = null;
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates a property builder.

lib/PhpParser/Builder/TraitUse.php

@@ -9,9 +9,9 @@ use PhpParser\Node\Stmt;
 
 class TraitUse implements Builder {
     /** @var Node\Name[] */
-    protected $traits = [];
+    protected array $traits = [];
     /** @var Stmt\TraitUseAdaptation[] */
-    protected $adaptations = [];
+    protected array $adaptations = [];
 
     /**
      * Creates a trait use builder.

lib/PhpParser/Builder/TraitUseAdaptation.php

@@ -13,18 +13,13 @@ class TraitUseAdaptation implements Builder {
     private const TYPE_ALIAS      = 1;
     private const TYPE_PRECEDENCE = 2;
 
-    /** @var int Type of building adaptation */
-    protected $type;
-    /** @var Node\Name|null */
-    protected $trait;
-    /** @var Node\Identifier */
-    protected $method;
-    /** @var int|null */
-    protected $modifier = null;
-    /** @var Node\Identifier|null */
-    protected $alias = null;
+    protected int $type;
+    protected ?Node\Name $trait;
+    protected Node\Identifier $method;
+    protected ?int $modifier = null;
+    protected ?Node\Identifier $alias = null;
     /** @var Node\Name[] */
-    protected $insteadof = [];
+    protected array $insteadof = [];
 
     /**
      * Creates a trait use adaptation builder.

lib/PhpParser/Builder/Trait_.php

@@ -8,18 +8,17 @@ use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
 class Trait_ extends Declaration {
-    /** @var string */
-    protected $name;
+    protected string $name;
     /** @var list<Stmt\TraitUse> */
-    protected $uses = [];
+    protected array $uses = [];
     /** @var list<Stmt\ClassConst> */
-    protected $constants = [];
+    protected array $constants = [];
     /** @var list<Stmt\Property> */
-    protected $properties = [];
+    protected array $properties = [];
     /** @var list<Stmt\ClassMethod> */
-    protected $methods = [];
+    protected array $methods = [];
     /** @var list<Node\AttributeGroup> */
-    protected $attributeGroups = [];
+    protected array $attributeGroups = [];
 
     /**
      * Creates an interface builder.

lib/PhpParser/Builder/Use_.php

@@ -8,18 +8,16 @@ use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
 class Use_ implements Builder {
-    /** @var Node\Name */
-    protected $name;
-    /** @var int */
-    protected $type;
-    /** @var string|null */
-    protected $alias = null;
+    protected Node\Name $name;
+    /** @var Stmt\Use_::TYPE_* */
+    protected int $type;
+    protected ?string $alias = null;
 
     /**
      * Creates a name use (alias) builder.
      *
      * @param Node\Name|string $name Name of the entity (namespace, class, function, constant) to alias
-     * @param int              $type One of the Stmt\Use_::TYPE_* constants
+     * @param Stmt\Use_::TYPE_* $type One of the Stmt\Use_::TYPE_* constants
      */
     public function __construct($name, int $type) {
         $this->name = BuilderHelpers::normalizeName($name);

lib/PhpParser/BuilderFactory.php

@@ -16,8 +16,6 @@ class BuilderFactory {
      *
      * @param string|Name $name Name of the attribute
      * @param array $args Attribute named arguments
-     *
-     * @return Node\Attribute
      */
     public function attribute($name, array $args = []): Node\Attribute {
         return new Node\Attribute(
@@ -213,8 +211,6 @@ class BuilderFactory {
      * Creates node a for a literal value.
      *
      * @param Expr|bool|null|int|float|string|array $value $value
-     *
-     * @return Expr
      */
     public function val($value): Expr {
         return BuilderHelpers::normalizeValue($value);
@@ -224,8 +220,6 @@ class BuilderFactory {
      * Creates variable node.
      *
      * @param string|Expr $name Name
-     *
-     * @return Expr\Variable
      */
     public function var($name): Expr\Variable {
         if (!\is_string($name) && !$name instanceof Expr) {
@@ -263,8 +257,6 @@ class BuilderFactory {
      *
      * @param string|Name|Expr $name Function name
      * @param array $args Function arguments
-     *
-     * @return Expr\FuncCall
      */
     public function funcCall($name, array $args = []): Expr\FuncCall {
         return new Expr\FuncCall(
@@ -279,8 +271,6 @@ class BuilderFactory {
      * @param Expr $var Variable the method is called on
      * @param string|Identifier|Expr $name Method name
      * @param array $args Method arguments
-     *
-     * @return Expr\MethodCall
      */
     public function methodCall(Expr $var, $name, array $args = []): Expr\MethodCall {
         return new Expr\MethodCall(
@@ -296,8 +286,6 @@ class BuilderFactory {
      * @param string|Name|Expr $class Class name
      * @param string|Identifier|Expr $name Method name
      * @param array $args Method arguments
-     *
-     * @return Expr\StaticCall
      */
     public function staticCall($class, $name, array $args = []): Expr\StaticCall {
         return new Expr\StaticCall(
@@ -312,8 +300,6 @@ class BuilderFactory {
      *
      * @param string|Name|Expr $class Class name
      * @param array $args Constructor arguments
-     *
-     * @return Expr\New_
      */
     public function new($class, array $args = []): Expr\New_ {
         return new Expr\New_(
@@ -326,8 +312,6 @@ class BuilderFactory {
      * Creates a constant fetch node.
      *
      * @param string|Name $name Constant name
-     *
-     * @return Expr\ConstFetch
      */
     public function constFetch($name): Expr\ConstFetch {
         return new Expr\ConstFetch(BuilderHelpers::normalizeName($name));
@@ -338,8 +322,6 @@ class BuilderFactory {
      *
      * @param Expr $var Variable holding object
      * @param string|Identifier|Expr $name Property name
-     *
-     * @return Expr\PropertyFetch
      */
     public function propertyFetch(Expr $var, $name): Expr\PropertyFetch {
         return new Expr\PropertyFetch($var, BuilderHelpers::normalizeIdentifierOrExpr($name));
@@ -350,8 +332,6 @@ class BuilderFactory {
      *
      * @param string|Name|Expr $class Class name
      * @param string|Identifier|Expr $name Constant name
-     *
-     * @return Expr\ClassConstFetch
      */
     public function classConstFetch($class, $name): Expr\ClassConstFetch {
         return new Expr\ClassConstFetch(
@@ -364,8 +344,6 @@ class BuilderFactory {
      * Creates nested Concat nodes from a list of expressions.
      *
      * @param Expr|string ...$exprs Expressions or literal strings
-     *
-     * @return Concat
      */
     public function concat(...$exprs): Concat {
         $numExprs = count($exprs);
@@ -382,7 +360,6 @@ class BuilderFactory {
 
     /**
      * @param string|Expr $expr
-     * @return Expr
      */
     private function normalizeStringExpr($expr): Expr {
         if ($expr instanceof Expr) {

lib/PhpParser/Comment.php

@@ -3,20 +3,13 @@
 namespace PhpParser;
 
 class Comment implements \JsonSerializable {
-    /** @var string string */
-    protected $text;
-    /** @var int */
-    protected $startLine;
-    /** @var int */
-    protected $startFilePos;
-    /** @var int */
-    protected $startTokenPos;
-    /** @var int */
-    protected $endLine;
-    /** @var int */
-    protected $endFilePos;
-    /** @var int */
-    protected $endTokenPos;
+    protected string $text;
+    protected int $startLine;
+    protected int $startFilePos;
+    protected int $startTokenPos;
+    protected int $endLine;
+    protected int $endFilePos;
+    protected int $endTokenPos;
 
     /**
      * Constructs a comment node.
@@ -103,39 +96,6 @@ class Comment implements \JsonSerializable {
         return $this->endTokenPos;
     }
 
-    /**
-     * Gets the line number the comment started on.
-     *
-     * @deprecated Use getStartLine() instead
-     *
-     * @return int Line number
-     */
-    public function getLine(): int {
-        return $this->startLine;
-    }
-
-    /**
-     * Gets the file offset the comment started on.
-     *
-     * @deprecated Use getStartFilePos() instead
-     *
-     * @return int File offset
-     */
-    public function getFilePos(): int {
-        return $this->startFilePos;
-    }
-
-    /**
-     * Gets the token offset the comment started on.
-     *
-     * @deprecated Use getStartTokenPos() instead
-     *
-     * @return int Token offset
-     */
-    public function getTokenPos(): int {
-        return $this->startTokenPos;
-    }
-
     /**
      * Gets the comment text.
      *
@@ -154,8 +114,6 @@ class Comment implements \JsonSerializable {
      * on all subsequent lines.
      *
      * Additionally, this normalizes CRLF newlines to LF newlines.
-     *
-     * @return string
      */
     public function getReformattedText(): string {
         $text = str_replace("\r\n", "\n", $this->text);
@@ -229,8 +187,7 @@ class Comment implements \JsonSerializable {
     }
 
     /**
-     * @return       array
-     * @psalm-return array{nodeType:string, text:mixed, line:mixed, filePos:mixed}
+     * @return array{nodeType:string, text:mixed, line:mixed, filePos:mixed}
      */
     public function jsonSerialize(): array {
         // Technically not a node, but we make it look like one anyway

lib/PhpParser/Error.php

@@ -3,10 +3,9 @@
 namespace PhpParser;
 
 class Error extends \RuntimeException {
-    /** @var string */
-    protected $rawMessage;
+    protected string $rawMessage;
     /** @var array<string, mixed> */
-    protected $attributes;
+    protected array $attributes;
 
     /**
      * Creates an Exception signifying a parse error.
@@ -90,8 +89,6 @@ class Error extends \RuntimeException {
      * Returns whether the error has start and end column information.
      *
      * For column information enable the startFilePos and endFilePos in the lexer options.
-     *
-     * @return bool
      */
     public function hasColumnInfo(): bool {
         return isset($this->attributes['startFilePos'], $this->attributes['endFilePos']);
@@ -101,7 +98,6 @@ class Error extends \RuntimeException {
      * Gets the start column (1-based) into the line where the error started.
      *
      * @param string $code Source code of the file
-     * @return int
      */
     public function getStartColumn(string $code): int {
         if (!$this->hasColumnInfo()) {
@@ -115,7 +111,6 @@ class Error extends \RuntimeException {
      * Gets the end column (1-based) into the line where the error ended.
      *
      * @param string $code Source code of the file
-     * @return int
      */
     public function getEndColumn(string $code): int {
         if (!$this->hasColumnInfo()) {

lib/PhpParser/ErrorHandler/Collecting.php

@@ -12,7 +12,7 @@ use PhpParser\ErrorHandler;
  */
 class Collecting implements ErrorHandler {
     /** @var Error[] Collected errors */
-    private $errors = [];
+    private array $errors = [];
 
     public function handleError(Error $error): void {
         $this->errors[] = $error;
@@ -29,8 +29,6 @@ class Collecting implements ErrorHandler {
 
     /**
      * Check whether there are any errors.
-     *
-     * @return bool
      */
     public function hasErrors(): bool {
         return !empty($this->errors);

lib/PhpParser/Internal/DiffElem.php

@@ -12,7 +12,7 @@ class DiffElem {
     public const TYPE_REPLACE = 3;
 
     /** @var int One of the TYPE_* constants */
-    public $type;
+    public int $type;
     /** @var mixed Is null for add operations */
     public $old;
     /** @var mixed Is null for remove operations */

lib/PhpParser/Internal/Differ.php

@@ -91,8 +91,6 @@ class Differ {
 
     /**
      * @param array<int, array<int, int>> $trace
-     * @param int $x
-     * @param int $y
      * @param T[] $old
      * @param T[] $new
      * @return DiffElem[]

lib/PhpParser/Internal/PrintableNewAnonClassNode.php

@@ -17,17 +17,17 @@ use PhpParser\Node\Expr;
  */
 class PrintableNewAnonClassNode extends Expr {
     /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
+    public array $attrGroups;
     /** @var int Modifiers */
-    public $flags;
+    public int $flags;
     /** @var (Node\Arg|Node\VariadicPlaceholder)[] Arguments */
-    public $args;
+    public array $args;
     /** @var null|Node\Name Name of extended class */
-    public $extends;
+    public ?Node\Name $extends;
     /** @var Node\Name[] Names of implemented interfaces */
-    public $implements;
+    public array $implements;
     /** @var Node\Stmt[] Statements */
-    public $stmts;
+    public array $stmts;
 
     /**
      * @param Node\AttributeGroup[] $attrGroups PHP attribute groups

lib/PhpParser/Internal/TokenPolyfill.php

@@ -17,13 +17,13 @@ if (\PHP_VERSION_ID >= 80000) {
  */
 class TokenPolyfill {
     /** @var int The ID of the token. Either a T_* constant of a character code < 256. */
-    public $id;
+    public int $id;
     /** @var string The textual content of the token. */
-    public $text;
+    public string $text;
     /** @var int The 1-based starting line of the token (or -1 if unknown). */
-    public $line;
+    public int $line;
     /** @var int The 0-based starting position of the token (or -1 if unknown). */
-    public $pos;
+    public int $pos;
 
     /** @var array<int, bool> Tokens ignored by the PHP parser. */
     private const IGNORABLE_TOKENS = [
@@ -33,8 +33,8 @@ class TokenPolyfill {
         \T_OPEN_TAG => true,
     ];
 
-    /** @var array<int, bool>|null Tokens that may be part of a T_NAME_* identifier. */
-    private static $identifierTokens;
+    /** @var array<int, bool> Tokens that may be part of a T_NAME_* identifier. */
+    private static array $identifierTokens;
 
     /**
      * Create a Token with the given ID and text, as well optional line and position information.
@@ -128,9 +128,6 @@ class TokenPolyfill {
         $line = 1;
         $pos = 0;
         $origTokens = \token_get_all($code, $flags);
-        if (\PHP_VERSION_ID < 70400) {
-            $origTokens = self::fixupBadCharacters($code, $origTokens);
-        }
 
         $numTokens = \count($origTokens);
         for ($i = 0; $i < $numTokens; $i++) {
@@ -217,38 +214,6 @@ class TokenPolyfill {
         return $tokens;
     }
 
-    /**
-     * Prior to PHP 7.4, token_get_all() simply dropped invalid characters from the token stream.
-     * Detect such cases and replace them with T_BAD_CHARACTER.
-     */
-    private static function fixupBadCharacters(string $code, array $origTokens): array {
-        $newTokens = [];
-        $pos = 0;
-        foreach ($origTokens as $token) {
-            $text = \is_string($token) ? $token : $token[1];
-            $len = \strlen($text);
-            if (substr($code, $pos, $len) !== $text) {
-                $nextPos = strpos($code, $text, $pos);
-                for ($i = $pos; $i < $nextPos; $i++) {
-                    // Don't bother including the line, we're not going to use it anyway.
-                    $newTokens[] = [\T_BAD_CHARACTER, $code[$i]];
-                }
-                $pos = $nextPos;
-            }
-            $pos += $len;
-            $newTokens[] = $token;
-        }
-
-        // Handle trailing invalid characters.
-        $codeLen = \strlen($code);
-        if ($pos !== $codeLen) {
-            for ($i = $pos; $i < $codeLen; $i++) {
-                $newTokens[] = [\T_BAD_CHARACTER, $code[$i]];
-            }
-        }
-        return $newTokens;
-    }
-
     /** Initialize private static state needed by tokenize(). */
     private static function init(): void {
         if (isset(self::$identifierTokens)) {

lib/PhpParser/Internal/TokenStream.php

@@ -11,9 +11,9 @@ use PhpParser\Token;
  */
 class TokenStream {
     /** @var Token[] Tokens (in PhpToken::tokenize() format) */
-    private $tokens;
+    private array $tokens;
     /** @var int[] Map from position to indentation */
-    private $indentMap;
+    private array $indentMap;
 
     /**
      * Create token stream instance.
@@ -30,8 +30,6 @@ class TokenStream {
      *
      * @param int $startPos Start position
      * @param int $endPos End position
-     *
-     * @return bool
      */
     public function haveParens(int $startPos, int $endPos): bool {
         return $this->haveTokenImmediatelyBefore($startPos, '(')
@@ -43,8 +41,6 @@ class TokenStream {
      *
      * @param int $startPos Start position
      * @param int $endPos End position
-     *
-     * @return bool
      */
     public function haveBraces(int $startPos, int $endPos): bool {
         return ($this->haveTokenImmediatelyBefore($startPos, '{')
@@ -199,12 +195,6 @@ class TokenStream {
         return false;
     }
 
-    public function haveBracesInRange(int $startPos, int $endPos): bool {
-        return $this->haveTokenInRange($startPos, $endPos, '{')
-            || $this->haveTokenInRange($startPos, $endPos, T_CURLY_OPEN)
-            || $this->haveTokenInRange($startPos, $endPos, '}');
-    }
-
     public function haveTagInRange(int $startPos, int $endPos): bool {
         return $this->haveTokenInRange($startPos, $endPos, \T_OPEN_TAG)
             || $this->haveTokenInRange($startPos, $endPos, \T_CLOSE_TAG);

lib/PhpParser/JsonDecoder.php

@@ -4,7 +4,7 @@ namespace PhpParser;
 
 class JsonDecoder {
     /** @var \ReflectionClass<Node>[] Node type to reflection class map */
-    private $reflectionClassCache;
+    private array $reflectionClassCache;
 
     /** @return mixed */
     public function decode(string $json) {

lib/PhpParser/Lexer.php

@@ -5,91 +5,37 @@ namespace PhpParser;
 require __DIR__ . '/compatibility_tokens.php';
 
 class Lexer {
-    /** @var string Code being tokenized */
-    protected $code;
-    /** @var list<Token> List of tokens */
-    protected $tokens;
-    /** @var int Current position in the token array */
-    protected $pos;
-    /** @var bool Whether the preceding closing PHP tag has a trailing newline */
-    protected $prevCloseTagHasNewline;
-    /** @var array<int, int> Map of tokens that should be dropped (like T_WHITESPACE) */
-    protected $dropTokens;
-
-    /** @var bool Whether to use the startLine attribute */
-    private $attributeStartLineUsed;
-    /** @var bool Whether to use the endLine attribute */
-    private $attributeEndLineUsed;
-    /** @var bool Whether to use the startTokenPos attribute */
-    private $attributeStartTokenPosUsed;
-    /** @var bool Whether to use the endTokenPos attribute */
-    private $attributeEndTokenPosUsed;
-    /** @var bool Whether to use the startFilePos attribute */
-    private $attributeStartFilePosUsed;
-    /** @var bool Whether to use the endFilePos attribute */
-    private $attributeEndFilePosUsed;
-    /** @var bool Whether to use the comments attribute */
-    private $attributeCommentsUsed;
-
     /**
-     * Creates a Lexer.
+     * Tokenize the provided source code.
      *
-     * @param array{usedAttributes?: string[]} $options Options array. Currently only the
-     *        'usedAttributes' option is supported, which is an array of attributes to add to the
-     *        AST nodes. Possible attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
-     *        'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the first three.
-     *        For more info see getNextToken() docs.
-     */
-    public function __construct(array $options = []) {
-        // map of tokens to drop while lexing (the map is only used for isset lookup,
-        // that's why the value is simply set to 1; the value is never actually used.)
-        $this->dropTokens = array_fill_keys(
-            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
-        );
-
-        $defaultAttributes = ['comments', 'startLine', 'endLine'];
-        $usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
-
-        // Create individual boolean properties to make these checks faster.
-        $this->attributeStartLineUsed = isset($usedAttributes['startLine']);
-        $this->attributeEndLineUsed = isset($usedAttributes['endLine']);
-        $this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
-        $this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
-        $this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
-        $this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
-        $this->attributeCommentsUsed = isset($usedAttributes['comments']);
-    }
-
-    /**
-     * Initializes the lexer for lexing the provided source code.
+     * The token array is in the same format as provided by the PhpToken::tokenize() method in
+     * PHP 8.0. The tokens are instances of PhpParser\Token, to abstract over a polyfill
+     * implementation in earlier PHP version.
      *
-     * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
-     * the getErrors() method.
+     * The token array is terminated by a sentinel token with token ID 0.
+     * The token array does not discard any tokens (i.e. whitespace and comments are included).
+     * The token position attributes are against this token array.
      *
-     * @param string $code The source code to lex
+     * @param string $code The source code to tokenize.
      * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
-     *                                        ErrorHandler\Throwing
+     *                                        ErrorHandler\Throwing.
+     * @return Token[] Tokens
      */
-    public function startLexing(string $code, ?ErrorHandler $errorHandler = null): void {
+    public function tokenize(string $code, ?ErrorHandler $errorHandler = null): array {
         if (null === $errorHandler) {
             $errorHandler = new ErrorHandler\Throwing();
         }
 
-        $this->code = $code; // keep the code around for __halt_compiler() handling
-        $this->pos = -1;
-
-        // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
-        // This ensures proper composability, because having a newline is the "safe" assumption.
-        $this->prevCloseTagHasNewline = true;
-
         $scream = ini_set('xdebug.scream', '0');
 
-        $this->tokens = @Token::tokenize($code);
-        $this->postprocessTokens($errorHandler);
+        $tokens = @Token::tokenize($code);
+        $this->postprocessTokens($tokens, $errorHandler);
 
         if (false !== $scream) {
             ini_set('xdebug.scream', $scream);
         }
+
+        return $tokens;
     }
 
     private function handleInvalidCharacter(Token $token, ErrorHandler $errorHandler): void {
@@ -117,33 +63,36 @@ class Lexer {
             && substr($token->text, -2) !== '*/';
     }
 
-    protected function postprocessTokens(ErrorHandler $errorHandler): void {
+    /**
+     * @param list<Token> $tokens
+     */
+    protected function postprocessTokens(array &$tokens, ErrorHandler $errorHandler): void {
         // This function reports errors (bad characters and unterminated comments) in the token
         // array, and performs certain canonicalizations:
         //  * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
         //    T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
         //  * Add a sentinel token with ID 0.
 
-        $numTokens = \count($this->tokens);
+        $numTokens = \count($tokens);
         if ($numTokens === 0) {
             // Empty input edge case: Just add the sentinel token.
-            $this->tokens[] = new Token(0, "\0", 1, 0);
+            $tokens[] = new Token(0, "\0", 1, 0);
             return;
         }
 
         for ($i = 0; $i < $numTokens; $i++) {
-            $token = $this->tokens[$i];
+            $token = $tokens[$i];
             if ($token->id === \T_BAD_CHARACTER) {
                 $this->handleInvalidCharacter($token, $errorHandler);
             }
 
             if ($token->id === \ord('&')) {
                 $next = $i + 1;
-                while (isset($this->tokens[$next]) && $this->tokens[$next]->id === \T_WHITESPACE) {
+                while (isset($tokens[$next]) && $tokens[$next]->id === \T_WHITESPACE) {
                     $next++;
                 }
-                $followedByVarOrVarArg = isset($this->tokens[$next]) &&
-                    $this->tokens[$next]->is([\T_VARIABLE, \T_ELLIPSIS]);
+                $followedByVarOrVarArg = isset($tokens[$next]) &&
+                    $tokens[$next]->is([\T_VARIABLE, \T_ELLIPSIS]);
                 $token->id = $followedByVarOrVarArg
                     ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
                     : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
@@ -151,7 +100,7 @@ class Lexer {
         }
 
         // Check for unterminated comment
-        $lastToken = $this->tokens[$numTokens - 1];
+        $lastToken = $tokens[$numTokens - 1];
         if ($this->isUnterminatedComment($lastToken)) {
             $errorHandler->handleError(new Error('Unterminated comment', [
                 'startLine' => $lastToken->line,
@@ -162,115 +111,6 @@ class Lexer {
         }
 
         // Add sentinel token.
-        $this->tokens[] = new Token(0, "\0", $lastToken->getEndLine(), $lastToken->getEndPos());
-    }
-
-    /**
-     * Fetches the next token.
-     *
-     * The available attributes are determined by the 'usedAttributes' option, which can
-     * be specified in the constructor. The following attributes are supported:
-     *
-     *  * 'comments'      => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
-     *                       representing all comments that occurred between the previous
-     *                       non-discarded token and the current one.
-     *  * 'startLine'     => Line in which the node starts.
-     *  * 'endLine'       => Line in which the node ends.
-     *  * 'startTokenPos' => Offset into the token array of the first token in the node.
-     *  * 'endTokenPos'   => Offset into the token array of the last token in the node.
-     *  * 'startFilePos'  => Offset into the code string of the first character that is part of the node.
-     *  * 'endFilePos'    => Offset into the code string of the last character that is part of the node.
-     *
-     * @param mixed $value           Variable to store token content in
-     * @param mixed $startAttributes Variable to store start attributes in
-     * @param mixed $endAttributes   Variable to store end attributes in
-     *
-     * @return int Token id
-     */
-    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null): int {
-        $startAttributes = [];
-        $endAttributes   = [];
-
-        while (1) {
-            $token = $this->tokens[++$this->pos];
-
-            if ($this->attributeStartLineUsed) {
-                $startAttributes['startLine'] = $token->line;
-            }
-            if ($this->attributeStartTokenPosUsed) {
-                $startAttributes['startTokenPos'] = $this->pos;
-            }
-            if ($this->attributeStartFilePosUsed) {
-                $startAttributes['startFilePos'] = $token->pos;
-            }
-
-            $id = $token->id;
-            if (isset($this->dropTokens[$id])) {
-                if (\T_COMMENT === $id || \T_DOC_COMMENT === $id) {
-                    if ($this->attributeCommentsUsed) {
-                        $comment = \T_DOC_COMMENT === $id
-                            ? new Comment\Doc($token->text, $token->line, $token->pos, $this->pos,
-                                $token->getEndLine(), $token->getEndPos() - 1, $this->pos)
-                            : new Comment($token->text, $token->line, $token->pos, $this->pos,
-                                $token->getEndLine(), $token->getEndPos() - 1, $this->pos);
-                        $startAttributes['comments'][] = $comment;
-                    }
-                }
-                continue;
-            }
-
-            $value = $token->text;
-            if (\T_CLOSE_TAG === $token->id) {
-                $this->prevCloseTagHasNewline = false !== strpos($value, "\n")
-                    || false !== strpos($value, "\r");
-            } elseif (\T_INLINE_HTML === $token->id) {
-                $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
-            }
-
-            // Fetch the end line/pos from the next token (if available) instead of recomputing it.
-            $nextToken = $this->tokens[$this->pos + 1] ?? null;
-            if ($this->attributeEndLineUsed) {
-                $endAttributes['endLine'] = $nextToken ? $nextToken->line : $token->getEndLine();
-            }
-            if ($this->attributeEndTokenPosUsed) {
-                $endAttributes['endTokenPos'] = $this->pos;
-            }
-            if ($this->attributeEndFilePosUsed) {
-                $endAttributes['endFilePos'] = ($nextToken ? $nextToken->pos : $token->getEndPos()) - 1;
-            }
-
-            return $id;
-        }
-    }
-
-    /**
-     * Returns the token array for current code.
-     *
-     * The token array is in the same format as provided by the PhpToken::tokenize() method in
-     * PHP 8.0. The tokens are instances of PhpParser\Token, to abstract over a polyfill
-     * implementation in earlier PHP version.
-     *
-     * The token array is terminated by a sentinel token with token ID 0.
-     * The token array does not discard any tokens (i.e. whitespace and comments are included).
-     * The token position attributes are against this token array.
-     *
-     * @return Token[] Array of tokens
-     */
-    public function getTokens(): array {
-        return $this->tokens;
-    }
-
-    /**
-     * Handles __halt_compiler() by returning the text after it.
-     *
-     * @return string Remaining text
-     */
-    public function handleHaltCompiler(): string {
-        // Prevent the lexer from returning any further tokens.
-        $nextToken = $this->tokens[$this->pos + 1];
-        $this->pos = \count($this->tokens) - 2;
-
-        // Return text after __halt_compiler.
-        return $nextToken->id === \T_INLINE_HTML ? $nextToken->text : '';
+        $tokens[] = new Token(0, "\0", $lastToken->getEndLine(), $lastToken->getEndPos());
     }
 }

lib/PhpParser/Lexer/Emulative.php

@@ -19,41 +19,28 @@ use PhpParser\Lexer\TokenEmulator\ReadonlyTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\ReverseEmulator;
 use PhpParser\Lexer\TokenEmulator\TokenEmulator;
 use PhpParser\PhpVersion;
+use PhpParser\Token;
 
 class Emulative extends Lexer {
     /** @var array{int, string, string}[] Patches used to reverse changes introduced in the code */
-    private $patches = [];
+    private array $patches = [];
 
     /** @var list<TokenEmulator> */
-    private $emulators = [];
+    private array $emulators = [];
 
-    /** @var PhpVersion */
-    private $targetPhpVersion;
-    /** @var PhpVersion */
-    private $hostPhpVersion;
+    private PhpVersion $targetPhpVersion;
+
+    private PhpVersion $hostPhpVersion;
 
     /**
-     * @param array{usedAttributes?: string[], phpVersion?: PhpVersion|string} $options Lexer options.
-     *        In addition to the usual options, accepts a 'phpVersion' (PhpVersion object or string)
-     *        that specifies the version to emulate. Defaults to newest supported.
+     * @param PhpVersion|null $phpVersion PHP version to emulate. Defaults to newest supported.
      */
-    public function __construct(array $options = []) {
-        $version = $options['phpVersion'] ?? PhpVersion::getNewestSupported();
-        if (!$version instanceof PhpVersion) {
-            $version = PhpVersion::fromString($version);
-        }
-        $this->targetPhpVersion = $version;
+    public function __construct(?PhpVersion $phpVersion = null) {
+        $this->targetPhpVersion = $phpVersion ?? PhpVersion::getNewestSupported();
         $this->hostPhpVersion = PhpVersion::getHostVersion();
-        unset($options['phpVersion']);
-
-        parent::__construct($options);
 
         $emulators = [
-            new FlexibleDocStringEmulator(),
-            new FnTokenEmulator(),
             new MatchTokenEmulator(),
-            new CoaleseEqualTokenEmulator(),
-            new NumericLiteralSeparatorEmulator(),
             new NullsafeTokenEmulator(),
             new AttributeEmulator(),
             new EnumTokenEmulator(),
@@ -74,15 +61,14 @@ class Emulative extends Lexer {
         }
     }
 
-    public function startLexing(string $code, ?ErrorHandler $errorHandler = null): void {
+    public function tokenize(string $code, ?ErrorHandler $errorHandler = null): array {
         $emulators = array_filter($this->emulators, function ($emulator) use ($code) {
             return $emulator->isEmulationNeeded($code);
         });
 
         if (empty($emulators)) {
             // Nothing to emulate, yay
-            parent::startLexing($code, $errorHandler);
-            return;
+            return parent::tokenize($code, $errorHandler);
         }
 
         if ($errorHandler === null) {
@@ -95,9 +81,9 @@ class Emulative extends Lexer {
         }
 
         $collector = new ErrorHandler\Collecting();
-        parent::startLexing($code, $collector);
+        $tokens = parent::tokenize($code, $collector);
         $this->sortPatches();
-        $this->fixupTokens();
+        $tokens = $this->fixupTokens($tokens);
 
         $errors = $collector->getErrors();
         if (!empty($errors)) {
@@ -108,8 +94,10 @@ class Emulative extends Lexer {
         }
 
         foreach ($emulators as $emulator) {
-            $this->tokens = $emulator->emulate($code, $this->tokens);
+            $tokens = $emulator->emulate($code, $tokens);
         }
+
+        return $tokens;
     }
 
     private function isForwardEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
@@ -130,9 +118,13 @@ class Emulative extends Lexer {
         });
     }
 
-    private function fixupTokens(): void {
+    /**
+     * @param list<Token> $tokens
+     * @return list<Token>
+     */
+    private function fixupTokens(array $tokens): array {
         if (\count($this->patches) === 0) {
-            return;
+            return $tokens;
         }
 
         // Load first patch
@@ -141,10 +133,12 @@ class Emulative extends Lexer {
 
         // We use a manual loop over the tokens, because we modify the array on the fly
         $posDelta = 0;
-        for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
-            $token = $this->tokens[$i];
+        $lineDelta = 0;
+        for ($i = 0, $c = \count($tokens); $i < $c; $i++) {
+            $token = $tokens[$i];
             $pos = $token->pos;
             $token->pos += $posDelta;
+            $token->line += $lineDelta;
             $localPosDelta = 0;
             $len = \strlen($token->text);
             while ($patchPos >= $pos && $patchPos < $pos + $len) {
@@ -152,7 +146,7 @@ class Emulative extends Lexer {
                 if ($patchType === 'remove') {
                     if ($patchPos === $pos && $patchTextLen === $len) {
                         // Remove token entirely
-                        array_splice($this->tokens, $i, 1, []);
+                        array_splice($tokens, $i, 1, []);
                         $i--;
                         $c--;
                     } else {
@@ -162,12 +156,14 @@ class Emulative extends Lexer {
                         );
                         $localPosDelta -= $patchTextLen;
                     }
+                    $lineDelta -= \substr_count($patchText, "\n");
                 } elseif ($patchType === 'add') {
                     // Insert into the token string
                     $token->text = substr_replace(
                         $token->text, $patchText, $patchPos - $pos + $localPosDelta, 0
                     );
                     $localPosDelta += $patchTextLen;
+                    $lineDelta += \substr_count($patchText, "\n");
                 } elseif ($patchType === 'replace') {
                     // Replace inside the token string
                     $token->text = substr_replace(
@@ -190,6 +186,7 @@ class Emulative extends Lexer {
 
             $posDelta += $localPosDelta;
         }
+        return $tokens;
     }
 
     /**

lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php

@@ -1,40 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\PhpVersion;
-use PhpParser\Token;
-
-final class CoaleseEqualTokenEmulator extends TokenEmulator {
-    public function getPhpVersion(): PhpVersion {
-        return PhpVersion::fromComponents(7, 4);
-    }
-
-    public function isEmulationNeeded(string $code): bool {
-        return strpos($code, '??=') !== false;
-    }
-
-    public function emulate(string $code, array $tokens): array {
-        // We need to manually iterate and manage a count because we'll change
-        // the tokens array on the way
-        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            $token = $tokens[$i];
-            if (isset($tokens[$i + 1])) {
-                if ($token->id === T_COALESCE && $tokens[$i + 1]->text === '=') {
-                    array_splice($tokens, $i, 2, [
-                        new Token(\T_COALESCE_EQUAL, '??=', $token->line, $token->pos),
-                    ]);
-                    $c--;
-                    continue;
-                }
-            }
-        }
-
-        return $tokens;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array {
-        // ??= was not valid code previously, don't bother.
-        return $tokens;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/FlexibleDocStringEmulator.php

@@ -1,71 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\PhpVersion;
-
-final class FlexibleDocStringEmulator extends TokenEmulator {
-    private const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
-/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
-(?:.*\r?\n)*?
-(?<indentation>\h*)\2(?![a-zA-Z0-9_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
-REGEX;
-
-    public function getPhpVersion(): PhpVersion {
-        return PhpVersion::fromComponents(7, 3);
-    }
-
-    public function isEmulationNeeded(string $code): bool {
-        return strpos($code, '<<<') !== false;
-    }
-
-    public function emulate(string $code, array $tokens): array {
-        // Handled by preprocessing + fixup.
-        return $tokens;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array {
-        // Not supported.
-        return $tokens;
-    }
-
-    public function preprocessCode(string $code, array &$patches): string {
-        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
-            // No heredoc/nowdoc found
-            return $code;
-        }
-
-        // Keep track of how much we need to adjust string offsets due to the modifications we
-        // already made
-        $posDelta = 0;
-        foreach ($matches as $match) {
-            $indentation = $match['indentation'][0];
-            $indentationStart = $match['indentation'][1];
-
-            $separator = $match['separator'][0];
-            $separatorStart = $match['separator'][1];
-
-            if ($indentation === '' && $separator !== '') {
-                // Ordinary heredoc/nowdoc
-                continue;
-            }
-
-            if ($indentation !== '') {
-                // Remove indentation
-                $indentationLen = strlen($indentation);
-                $code = substr_replace($code, '', $indentationStart + $posDelta, $indentationLen);
-                $patches[] = [$indentationStart + $posDelta, 'add', $indentation];
-                $posDelta -= $indentationLen;
-            }
-
-            if ($separator === '') {
-                // Insert newline as separator
-                $code = substr_replace($code, "\n", $separatorStart + $posDelta, 0);
-                $patches[] = [$separatorStart + $posDelta, 'remove', "\n"];
-                $posDelta += 1;
-            }
-        }
-
-        return $code;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php

@@ -1,19 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\PhpVersion;
-
-final class FnTokenEmulator extends KeywordEmulator {
-    public function getPhpVersion(): PhpVersion {
-        return PhpVersion::fromComponents(7, 4);
-    }
-
-    public function getKeywordString(): string {
-        return 'fn';
-    }
-
-    public function getKeywordToken(): int {
-        return \T_FN;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -1,95 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\PhpVersion;
-use PhpParser\Token;
-
-final class NumericLiteralSeparatorEmulator extends TokenEmulator {
-    private const BIN = '(?:0b[01]+(?:_[01]+)*)';
-    private const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
-    private const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
-    private const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
-    private const EXP = '(?:e[+-]?' . self::DEC . ')';
-    private const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
-    private const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
-
-    public function getPhpVersion(): PhpVersion {
-        return PhpVersion::fromComponents(7, 4);
-    }
-
-    public function isEmulationNeeded(string $code): bool {
-        return preg_match('~[0-9]_[0-9]~', $code)
-            || preg_match('~0x[0-9a-f]+_[0-9a-f]~i', $code);
-    }
-
-    public function emulate(string $code, array $tokens): array {
-        // We need to manually iterate and manage a count because we'll change
-        // the tokens array on the way
-        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            $token = $tokens[$i];
-            $tokenLen = \strlen($token->text);
-
-            if ($token->id !== \T_LNUMBER && $token->id !== \T_DNUMBER) {
-                continue;
-            }
-
-            $res = preg_match(self::NUMBER, $code, $matches, 0, $token->pos);
-            assert($res, "No number at number token position");
-
-            $match = $matches[0];
-            $matchLen = \strlen($match);
-            if ($matchLen === $tokenLen) {
-                // Original token already holds the full number.
-                continue;
-            }
-
-            $tokenKind = $this->resolveIntegerOrFloatToken($match);
-            $newTokens = [new Token($tokenKind, $match, $token->line, $token->pos)];
-
-            $numTokens = 1;
-            $len = $tokenLen;
-            while ($matchLen > $len) {
-                $nextToken = $tokens[$i + $numTokens];
-                $nextTokenText = $nextToken->text;
-                $nextTokenLen = \strlen($nextTokenText);
-
-                $numTokens++;
-                if ($matchLen < $len + $nextTokenLen) {
-                    // Split trailing characters into a partial token.
-                    $partialText = substr($nextTokenText, $matchLen - $len);
-                    $newTokens[] = new Token($nextToken->id, $partialText, $nextToken->line, $nextToken->pos);
-                    break;
-                }
-
-                $len += $nextTokenLen;
-            }
-
-            array_splice($tokens, $i, $numTokens, $newTokens);
-            $c -= $numTokens - \count($newTokens);
-        }
-
-        return $tokens;
-    }
-
-    private function resolveIntegerOrFloatToken(string $str): int {
-        $str = str_replace('_', '', $str);
-
-        if (stripos($str, '0b') === 0) {
-            $num = bindec($str);
-        } elseif (stripos($str, '0x') === 0) {
-            $num = hexdec($str);
-        } elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
-            $num = octdec($str);
-        } else {
-            $num = +$str;
-        }
-
-        return is_float($num) ? T_DNUMBER : T_LNUMBER;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array {
-        // Numeric separators were not legal code previously, don't bother.
-        return $tokens;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/ReverseEmulator.php

@@ -9,7 +9,7 @@ use PhpParser\PhpVersion;
  */
 final class ReverseEmulator extends TokenEmulator {
     /** @var TokenEmulator Inner emulator */
-    private $emulator;
+    private TokenEmulator $emulator;
 
     public function __construct(TokenEmulator $emulator) {
         $this->emulator = $emulator;

lib/PhpParser/NameContext.php

@@ -8,16 +8,16 @@ use PhpParser\Node\Stmt;
 
 class NameContext {
     /** @var null|Name Current namespace */
-    protected $namespace;
+    protected ?Name $namespace;
 
     /** @var Name[][] Map of format [aliasType => [aliasName => originalName]] */
-    protected $aliases = [];
+    protected array $aliases = [];
 
     /** @var Name[][] Same as $aliases but preserving original case */
-    protected $origAliases = [];
+    protected array $origAliases = [];
 
     /** @var ErrorHandler Error handler */
-    protected $errorHandler;
+    protected ErrorHandler $errorHandler;
 
     /**
      * Create a name context.
@@ -49,7 +49,7 @@ class NameContext {
      *
      * @param Name $name Original name
      * @param string $aliasName Aliased name
-     * @param int    $type        One of Stmt\Use_::TYPE_*
+     * @param Stmt\Use_::TYPE_* $type One of Stmt\Use_::TYPE_*
      * @param array<string, mixed> $errorAttrs Attributes to use to report an error
      */
     public function addAlias(Name $name, string $aliasName, int $type, array $errorAttrs = []): void {
@@ -94,7 +94,7 @@ class NameContext {
      * Get resolved name.
      *
      * @param Name $name Name to resolve
-     * @param int  $type One of Stmt\Use_::TYPE_{FUNCTION|CONSTANT}
+     * @param Stmt\Use_::TYPE_* $type One of Stmt\Use_::TYPE_{FUNCTION|CONSTANT}
      *
      * @return null|Name Resolved name, or null if static resolution is not possible
      */
@@ -149,7 +149,7 @@ class NameContext {
      * Get possible ways of writing a fully qualified name (e.g., by making use of aliases).
      *
      * @param string $name Fully-qualified name (without leading namespace separator)
-     * @param int    $type One of Stmt\Use_::TYPE_*
+     * @param Stmt\Use_::TYPE_* $type One of Stmt\Use_::TYPE_*
      *
      * @return Name[] Possible representations of the name
      */
@@ -205,7 +205,7 @@ class NameContext {
      * Get shortest representation of this fully-qualified name.
      *
      * @param string $name Fully-qualified name (without leading namespace separator)
-     * @param int    $type One of Stmt\Use_::TYPE_*
+     * @param Stmt\Use_::TYPE_* $type One of Stmt\Use_::TYPE_*
      *
      * @return Name Shortest representation
      */

lib/PhpParser/Node.php

@@ -21,6 +21,8 @@ interface Node {
      * Gets line the node started in (alias of getStartLine).
      *
      * @return int Start line (or -1 if not available)
+     *
+     * @deprecated Use getStartLine() instead
      */
     public function getLine(): int;
 
@@ -110,24 +112,18 @@ interface Node {
     /**
      * Sets an attribute on a node.
      *
-     * @param string $key
      * @param mixed $value
      */
     public function setAttribute(string $key, $value): void;
 
     /**
      * Returns whether an attribute exists.
-     *
-     * @param string $key
-     *
-     * @return bool
      */
     public function hasAttribute(string $key): bool;
 
     /**
      * Returns the value of an attribute.
      *
-     * @param string $key
      * @param mixed $default
      *
      * @return mixed

lib/PhpParser/Node/Arg.php

@@ -6,13 +6,13 @@ use PhpParser\NodeAbstract;
 
 class Arg extends NodeAbstract {
     /** @var Identifier|null Parameter name (for named parameters) */
-    public $name;
+    public ?Identifier $name;
     /** @var Expr Value to pass */
-    public $value;
+    public Expr $value;
     /** @var bool Whether to pass by ref */
-    public $byRef;
+    public bool $byRef;
     /** @var bool Whether to unpack the argument */
-    public $unpack;
+    public bool $unpack;
 
     /**
      * Constructs a function call argument node.

lib/PhpParser/Node/ArrayItem.php

@@ -6,13 +6,13 @@ use PhpParser\NodeAbstract;
 
 class ArrayItem extends NodeAbstract {
     /** @var null|Expr Key */
-    public $key;
+    public ?Expr $key;
     /** @var Expr Value */
-    public $value;
+    public Expr $value;
     /** @var bool Whether to assign by reference */
-    public $byRef;
+    public bool $byRef;
     /** @var bool Whether to unpack the argument */
-    public $unpack;
+    public bool $unpack;
 
     /**
      * Constructs an array item node.

lib/PhpParser/Node/Attribute.php

@@ -7,10 +7,10 @@ use PhpParser\NodeAbstract;
 
 class Attribute extends NodeAbstract {
     /** @var Name Attribute name */
-    public $name;
+    public Name $name;
 
     /** @var list<Arg> Attribute arguments */
-    public $args;
+    public array $args;
 
     /**
      * @param Node\Name $name Attribute name

lib/PhpParser/Node/AttributeGroup.php

@@ -6,7 +6,7 @@ use PhpParser\NodeAbstract;
 
 class AttributeGroup extends NodeAbstract {
     /** @var Attribute[] Attributes */
-    public $attrs;
+    public array $attrs;
 
     /**
      * @param Attribute[] $attrs PHP attributes

lib/PhpParser/Node/ClosureUse.php

@@ -6,9 +6,9 @@ use PhpParser\NodeAbstract;
 
 class ClosureUse extends NodeAbstract {
     /** @var Expr\Variable Variable to use */
-    public $var;
+    public Expr\Variable $var;
     /** @var bool Whether to use by reference */
-    public $byRef;
+    public bool $byRef;
 
     /**
      * Constructs a closure use node.

lib/PhpParser/Node/Const_.php

@@ -6,12 +6,12 @@ use PhpParser\NodeAbstract;
 
 class Const_ extends NodeAbstract {
     /** @var Identifier Name */
-    public $name;
+    public Identifier $name;
     /** @var Expr Value */
-    public $value;
+    public Expr $value;
 
     /** @var Name|null Namespaced name (if using NameResolver) */
-    public $namespacedName;
+    public ?Name $namespacedName;
 
     /**
      * Constructs a const node for use in class const and const statements.

lib/PhpParser/Node/DeclareItem.php

@@ -7,9 +7,9 @@ use PhpParser\NodeAbstract;
 
 class DeclareItem extends NodeAbstract {
     /** @var Node\Identifier Key */
-    public $key;
+    public Identifier $key;
     /** @var Node\Expr Value */
-    public $value;
+    public Expr $value;
 
     /**
      * Constructs a declare key=>value pair node.

lib/PhpParser/Node/Expr/ArrayDimFetch.php

@@ -6,9 +6,9 @@ use PhpParser\Node\Expr;
 
 class ArrayDimFetch extends Expr {
     /** @var Expr Variable */
-    public $var;
+    public Expr $var;
     /** @var null|Expr Array index / dim */
-    public $dim;
+    public ?Expr $dim;
 
     /**
      * Constructs an array index fetch node.

lib/PhpParser/Node/Expr/Array_.php

@@ -11,7 +11,7 @@ class Array_ extends Expr {
     public const KIND_SHORT = 2; // [] syntax
 
     /** @var ArrayItem[] Items */
-    public $items;
+    public array $items;
 
     /**
      * Constructs an array node.

lib/PhpParser/Node/Expr/ArrowFunction.php

@@ -7,22 +7,22 @@ use PhpParser\Node\Expr;
 use PhpParser\Node\FunctionLike;
 
 class ArrowFunction extends Expr implements FunctionLike {
-    /** @var bool */
-    public $static;
+    /** @var bool Whether the closure is static */
+    public bool $static;
 
-    /** @var bool */
-    public $byRef;
+    /** @var bool Whether to return by reference */
+    public bool $byRef;
 
     /** @var Node\Param[] */
-    public $params = [];
+    public array $params = [];
 
     /** @var null|Node\Identifier|Node\Name|Node\ComplexType */
-    public $returnType;
+    public ?Node $returnType;
 
-    /** @var Expr */
-    public $expr;
+    /** @var Expr Expression body */
+    public Expr $expr;
     /** @var Node\AttributeGroup[] */
-    public $attrGroups;
+    public array $attrGroups;
 
     /**
      * @param array{

lib/PhpParser/Node/Expr/Assign.php

@@ -6,9 +6,9 @@ use PhpParser\Node\Expr;
 
 class Assign extends Expr {
     /** @var Expr Variable */
-    public $var;
+    public Expr $var;
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs an assignment node.

lib/PhpParser/Node/Expr/AssignOp.php

@@ -6,9 +6,9 @@ use PhpParser\Node\Expr;
 
 abstract class AssignOp extends Expr {
     /** @var Expr Variable */
-    public $var;
+    public Expr $var;
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs a compound assignment operation node.

lib/PhpParser/Node/Expr/AssignRef.php

@@ -6,9 +6,9 @@ use PhpParser\Node\Expr;
 
 class AssignRef extends Expr {
     /** @var Expr Variable reference is assigned to */
-    public $var;
+    public Expr $var;
     /** @var Expr Variable which is referenced */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs an assignment node.

lib/PhpParser/Node/Expr/BinaryOp.php

@@ -6,9 +6,9 @@ use PhpParser\Node\Expr;
 
 abstract class BinaryOp extends Expr {
     /** @var Expr The left hand side expression */
-    public $left;
+    public Expr $left;
     /** @var Expr The right hand side expression */
-    public $right;
+    public Expr $right;
 
     /**
      * Constructs a binary operator node.
@@ -32,8 +32,6 @@ abstract class BinaryOp extends Expr {
      *
      * In the case there are multiple possible sigils for an operator, this method does not
      * necessarily return the one used in the parsed code.
-     *
-     * @return string
      */
     abstract public function getOperatorSigil(): string;
 }

lib/PhpParser/Node/Expr/BitwiseNot.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class BitwiseNot extends Expr {
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs a bitwise not node.

lib/PhpParser/Node/Expr/BooleanNot.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class BooleanNot extends Expr {
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs a boolean not node.

lib/PhpParser/Node/Expr/CallLike.php

@@ -19,12 +19,8 @@ abstract class CallLike extends Expr {
      * Returns whether this call expression is actually a first class callable.
      */
     public function isFirstClassCallable(): bool {
-        foreach ($this->getRawArgs() as $arg) {
-            if ($arg instanceof VariadicPlaceholder) {
-                return true;
-            }
-        }
-        return false;
+        $rawArgs = $this->getRawArgs();
+        return count($rawArgs) === 1 && current($rawArgs) instanceof VariadicPlaceholder;
     }
 
     /**

lib/PhpParser/Node/Expr/Cast.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 abstract class Cast extends Expr {
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs a cast node.

lib/PhpParser/Node/Expr/ClassConstFetch.php

@@ -2,15 +2,16 @@
 
 namespace PhpParser\Node\Expr;
 
+use PhpParser\Node;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
 
 class ClassConstFetch extends Expr {
     /** @var Name|Expr Class name */
-    public $class;
+    public Node $class;
     /** @var Identifier|Expr|Error Constant name */
-    public $name;
+    public Node $name;
 
     /**
      * Constructs a class const fetch node.
@@ -19,7 +20,7 @@ class ClassConstFetch extends Expr {
      * @param string|Identifier|Expr|Error $name Constant name
      * @param array<string, mixed> $attributes Additional attributes
      */
-    public function __construct($class, $name, array $attributes = []) {
+    public function __construct(Node $class, $name, array $attributes = []) {
         $this->attributes = $attributes;
         $this->class = $class;
         $this->name = \is_string($name) ? new Identifier($name) : $name;

lib/PhpParser/Node/Expr/Clone_.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class Clone_ extends Expr {
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs a clone node.

lib/PhpParser/Node/Expr/Closure.php

@@ -9,19 +9,19 @@ use PhpParser\Node\FunctionLike;
 
 class Closure extends Expr implements FunctionLike {
     /** @var bool Whether the closure is static */
-    public $static;
+    public bool $static;
     /** @var bool Whether to return by reference */
-    public $byRef;
+    public bool $byRef;
     /** @var Node\Param[] Parameters */
-    public $params;
+    public array $params;
     /** @var ClosureUse[] use()s */
-    public $uses;
+    public array $uses;
     /** @var null|Node\Identifier|Node\Name|Node\ComplexType Return type */
-    public $returnType;
+    public ?Node $returnType;
     /** @var Node\Stmt[] Statements */
-    public $stmts;
+    public array $stmts;
     /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
+    public array $attrGroups;
 
     /**
      * Constructs a lambda function node.

lib/PhpParser/Node/Expr/ConstFetch.php

@@ -7,7 +7,7 @@ use PhpParser\Node\Name;
 
 class ConstFetch extends Expr {
     /** @var Name Constant name */
-    public $name;
+    public Name $name;
 
     /**
      * Constructs a const fetch node.

lib/PhpParser/Node/Expr/Empty_.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class Empty_ extends Expr {
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs an empty() node.

lib/PhpParser/Node/Expr/ErrorSuppress.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class ErrorSuppress extends Expr {
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs an error suppress node.

lib/PhpParser/Node/Expr/Eval_.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class Eval_ extends Expr {
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
 
     /**
      * Constructs an eval() node.

lib/PhpParser/Node/Expr/Exit_.php

@@ -10,7 +10,7 @@ class Exit_ extends Expr {
     public const KIND_DIE = 2;
 
     /** @var null|Expr Expression */
-    public $expr;
+    public ?Expr $expr;
 
     /**
      * Constructs an exit() node.

lib/PhpParser/Node/Expr/FuncCall.php

@@ -7,9 +7,9 @@ use PhpParser\Node\Expr;
 
 class FuncCall extends CallLike {
     /** @var Node\Name|Expr Function name */
-    public $name;
+    public Node $name;
     /** @var array<Node\Arg|Node\VariadicPlaceholder> Arguments */
-    public $args;
+    public array $args;
 
     /**
      * Constructs a function call node.
@@ -18,7 +18,7 @@ class FuncCall extends CallLike {
      * @param array<Node\Arg|Node\VariadicPlaceholder> $args Arguments
      * @param array<string, mixed> $attributes Additional attributes
      */
-    public function __construct($name, array $args = [], array $attributes = []) {
+    public function __construct(Node $name, array $args = [], array $attributes = []) {
         $this->attributes = $attributes;
         $this->name = $name;
         $this->args = $args;

lib/PhpParser/Node/Expr/Include_.php

@@ -11,9 +11,9 @@ class Include_ extends Expr {
     public const TYPE_REQUIRE_ONCE = 4;
 
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
     /** @var int Type of include */
-    public $type;
+    public int $type;
 
     /**
      * Constructs an include node.

lib/PhpParser/Node/Expr/Instanceof_.php

@@ -2,14 +2,15 @@
 
 namespace PhpParser\Node\Expr;
 
+use PhpParser\Node;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Name;
 
 class Instanceof_ extends Expr {
     /** @var Expr Expression */
-    public $expr;
+    public Expr $expr;
     /** @var Name|Expr Class name */
-    public $class;
+    public Node $class;
 
     /**
      * Constructs an instanceof check node.
@@ -18,7 +19,7 @@ class Instanceof_ extends Expr {
      * @param Name|Expr $class Class name
      * @param array<string, mixed> $attributes Additional attributes
      */
-    public function __construct(Expr $expr, $class, array $attributes = []) {
+    public function __construct(Expr $expr, Node $class, array $attributes = []) {
         $this->attributes = $attributes;
         $this->expr = $expr;
         $this->class = $class;

lib/PhpParser/Node/Expr/Isset_.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class Isset_ extends Expr {
     /** @var Expr[] Variables */
-    public $vars;
+    public array $vars;
 
     /**
      * Constructs an array node.

lib/PhpParser/Node/Expr/List_.php

@@ -11,7 +11,7 @@ class List_ extends Expr {
     public const KIND_ARRAY = 2; // [] syntax
 
     /** @var (ArrayItem|null)[] List of items to assign to */
-    public $items;
+    public array $items;
 
     /**
      * Constructs a list() destructuring node.

lib/PhpParser/Node/Expr/Match_.php

@@ -6,13 +6,15 @@ use PhpParser\Node;
 use PhpParser\Node\MatchArm;
 
 class Match_ extends Node\Expr {
-    /** @var Node\Expr */
-    public $cond;
+    /** @var Node\Expr Condition */
+    public Node\Expr $cond;
     /** @var MatchArm[] */
-    public $arms;
+    public array $arms;
 
     /**
+     * @param Node\Expr $cond Condition
      * @param MatchArm[] $arms
+     * @param array<string, mixed> $attributes Additional attributes
      */
     public function __construct(Node\Expr $cond, array $arms = [], array $attributes = []) {
         $this->attributes = $attributes;

lib/PhpParser/Node/Expr/MethodCall.php

@@ -2,6 +2,7 @@
 
 namespace PhpParser\Node\Expr;
 
+use PhpParser\Node;
 use PhpParser\Node\Arg;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
@@ -9,11 +10,11 @@ use PhpParser\Node\VariadicPlaceholder;
 
 class MethodCall extends CallLike {
     /** @var Expr Variable holding object */
-    public $var;
+    public Expr $var;
     /** @var Identifier|Expr Method name */
-    public $name;
+    public Node $name;
     /** @var array<Arg|VariadicPlaceholder> Arguments */
-    public $args;
+    public array $args;
 
     /**
      * Constructs a function call node.

lib/PhpParser/Node/Expr/New_.php

@@ -9,9 +9,9 @@ use PhpParser\Node\VariadicPlaceholder;
 
 class New_ extends CallLike {
     /** @var Node\Name|Expr|Node\Stmt\Class_ Class name */
-    public $class;
+    public Node $class;
     /** @var array<Arg|VariadicPlaceholder> Arguments */
-    public $args;
+    public array $args;
 
     /**
      * Constructs a function call node.
@@ -20,7 +20,7 @@ class New_ extends CallLike {
      * @param array<Arg|VariadicPlaceholder> $args Arguments
      * @param array<string, mixed> $attributes Additional attributes
      */
-    public function __construct($class, array $args = [], array $attributes = []) {
+    public function __construct(Node $class, array $args = [], array $attributes = []) {
         $this->attributes = $attributes;
         $this->class = $class;
         $this->args = $args;

lib/PhpParser/Node/Expr/NullsafeMethodCall.php

@@ -2,6 +2,7 @@
 
 namespace PhpParser\Node\Expr;
 
+use PhpParser\Node;
 use PhpParser\Node\Arg;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
@@ -9,11 +10,11 @@ use PhpParser\Node\VariadicPlaceholder;
 
 class NullsafeMethodCall extends CallLike {
     /** @var Expr Variable holding object */
-    public $var;
+    public Expr $var;
     /** @var Identifier|Expr Method name */
-    public $name;
+    public Node $name;
     /** @var array<Arg|VariadicPlaceholder> Arguments */
-    public $args;
+    public array $args;
 
     /**
      * Constructs a nullsafe method call node.

lib/PhpParser/Node/Expr/NullsafePropertyFetch.php

@@ -2,14 +2,15 @@
 
 namespace PhpParser\Node\Expr;
 
+use PhpParser\Node;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
 
 class NullsafePropertyFetch extends Expr {
     /** @var Expr Variable holding object */
-    public $var;
+    public Expr $var;
     /** @var Identifier|Expr Property name */
-    public $name;
+    public Node $name;
 
     /**
      * Constructs a nullsafe property fetch node.

lib/PhpParser/Node/Expr/PostDec.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class PostDec extends Expr {
     /** @var Expr Variable */
-    public $var;
+    public Expr $var;
 
     /**
      * Constructs a post decrement node.

lib/PhpParser/Node/Expr/PostInc.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class PostInc extends Expr {
     /** @var Expr Variable */
-    public $var;
+    public Expr $var;
 
     /**
      * Constructs a post increment node.

lib/PhpParser/Node/Expr/PreDec.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class PreDec extends Expr {
     /** @var Expr Variable */
-    public $var;
+    public Expr $var;
 
     /**
      * Constructs a pre decrement node.

lib/PhpParser/Node/Expr/PreInc.php

@@ -6,7 +6,7 @@ use PhpParser\Node\Expr;
 
 class PreInc extends Expr {
     /** @var Expr Variable */
-    public $var;
+    public Expr $var;
 
     /**
      * Constructs a pre increment node.