Release PHP-Parser 5.0.0-alpha1

Also highlights that the list insertion entry for Expr\Match
was not used.

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*.y]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4

.gitattributes

@@ -0,0 +1,11 @@
+/.github         export-ignore
+/doc             export-ignore
+/test            export-ignore
+/test_old        export-ignore
+.editorconfig    export-ignore
+.gitattributes   export-ignore
+.gitignore       export-ignore
+CHANGELOG.md     export-ignore
+CONTRIBUTING.md  export-ignore
+phpunit.xml.dist export-ignore
+UPGRADE-*.md     export-ignore

.github/workflows/main.yml

@@ -0,0 +1,86 @@
+# https://help.github.com/en/categories/automating-your-workflow-with-github-actions
+name: Main
+on:
+  push:
+  pull_request:
+
+jobs:
+  tests_coverage:
+    runs-on: "ubuntu-latest"
+    name: "PHP 7.1 Unit Tests (with coverage)"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "xdebug"
+          php-version: "7.1"
+          tools: composer:v2
+      - name: "Install dependencies"
+        run: |
+          composer require php-coveralls/php-coveralls:^2.2 --dev --no-update
+          composer update --no-progress --prefer-dist
+      - name: "Tests"
+        run: "php vendor/bin/phpunit --coverage-clover build/logs/clover.xml"
+      - name: Coveralls
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: "php vendor/bin/php-coveralls"
+        if: ${{ success() }}
+  tests:
+    runs-on: "ubuntu-latest"
+    name: "PHP ${{ matrix.php-version }} Unit Tests"
+    strategy:
+      matrix:
+        php-version:
+          - "7.2"
+          - "7.3"
+          - "7.4"
+          - "8.0"
+          - "8.1"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "${{ matrix.php-version }}"
+          tools: composer:v2
+      - name: "Install dependencies"
+        run: "composer update --no-progress --prefer-dist ${{ matrix.flags }}"
+      - name: "PHPUnit"
+        run: "php vendor/bin/phpunit"
+  test_old_73_80:
+    runs-on: "ubuntu-latest"
+    name: "PHP 7.3 Code on PHP 8.0 Integration Tests"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "8.0"
+          tools: composer:v2
+      - name: "Install PHP 8 dependencies"
+        run: "composer update --no-progress --prefer-dist"
+      - name: "Tests"
+        run: "test_old/run-php-src.sh 7.3.21"
+  test_old_80_70:
+    runs-on: "ubuntu-latest"
+    name: "PHP 8.1 Code on PHP 7.1 Integration Tests"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v2"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "7.1"
+          tools: composer:v2
+      - name: "Install PHP 8 dependencies"
+        run: "composer update --no-progress --prefer-dist"
+      - name: "Tests"
+        run: "test_old/run-php-src.sh 8.1.6"

.gitignore

@@ -2,3 +2,4 @@ vendor/
 composer.lock
 grammar/kmyacc.exe
 grammar/y.output
+.phpunit.result.cache

.php-cs-fixer.dist.php

@@ -0,0 +1,26 @@
+<?php
+
+$finder = PhpCsFixer\Finder::create()
+    ->exclude('PhpParser/Parser')
+    ->in(__DIR__ . '/lib')
+    ->in(__DIR__ . '/test')
+    ->in(__DIR__ . '/grammar')
+;
+
+$config = new PhpCsFixer\Config();
+return $config->setRiskyAllowed(true)
+    ->setRules([
+        '@PSR12' => true,
+        // We use PSR12 with consistent brace placement.
+        'curly_braces_position' => [
+            'functions_opening_brace' => 'same_line',
+            'classes_opening_brace' => 'same_line',
+        ],
+        // declare(strict_types=1) on the same line as <?php.
+        'blank_line_after_opening_tag' => false,
+        'declare_strict_types' => true,
+        // Keep argument formatting for now.
+        'method_argument_space' => ['on_multiline' => 'ignore'],
+    ])
+    ->setFinder($finder)
+;

.travis.yml

@@ -1,30 +0,0 @@
-language: php
-dist: xenial
-sudo: false
-
-cache:
-  directories:
-    - $HOME/.composer/cache
-
-php:
-  - 7.0
-  - 7.1
-  - 7.2
-  - 7.3
-  - nightly
-
-install:
-  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then composer require satooshi/php-coveralls '~1.0'; fi
-  - composer install --prefer-dist
-
-matrix:
-  allow_failures:
-    - php: nightly
-  fast_finish: true
-
-script:
-  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then vendor/bin/phpunit --coverage-clover build/logs/clover.xml; else vendor/bin/phpunit; fi
-  - if [ $TRAVIS_PHP_VERSION = '7.2' ]; then test_old/run-php-src.sh; fi
-
-after_success:
-  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then php vendor/bin/coveralls; fi

CHANGELOG.md

@@ -1,8 +1,389 @@
-Version 4.2.1-dev
+Version 5.0.0-dev
 -----------------
 
 Nothing yet.
 
+Version 5.0.0-alpha1 (2022-09-04)
+---------------------------------
+
+See UPGRADE-5.0 for detailed migration instructions.
+
+### Changed
+
+* PHP 7.1 is now required to run PHP-Parser.
+* Formatting of the standard pretty printer has been adjusted to match PSR-12 more closely.
+* The internal token representation now uses a `PhpParser\Token` class, which is compatible with
+  PHP 8 token representation (`PhpToken`).
+* Destructuring is now always represented using `Expr\List_` nodes, even if it uses `[]` syntax.
+* Renamed a number of node classes, and moved things that were not real expressions/statements
+  outside the `Expr`/`Stmt` hierarchy. Compatibility shims for the old names have been retained.
+
+### Added
+
+* Added `PhpVersion` class, which is accepted in a number of places (e.g. ParserFactory, Parser,
+  Lexer, PrettyPrinter) and gives more precise control over the PHP version being targeted.
+* Added PHP 8 parser though it only differs from the PHP 7 parser in concatenation precedence.
+* Added `Parser::getLexer()` method.
+* Added a `Modifiers` class, as a replacement for `Stmt\Class_::MODIFIER_*`.
+* Added support for returning an array or `REMOVE_NODE` from `NodeVisitor::enterNode()`.
+
+### Removed
+
+* The PHP 5 parser has been removed. The PHP 7 parser has been adjusted to deal with PHP 5 code
+  more gracefully.
+
+Version 4.15.1 (2022-09-04)
+---------------------------
+
+### Fixed
+
+* Fixed formatting preservation when adding *multiple* attributes to a class/method/etc that
+  previously had none. This fixes a regression in the 4.15.0 release.
+
+Version 4.15.0 (2022-09-03)
+---------------------------
+
+### Added
+
+* PHP 8.2: Added support for `true` type.
+* PHP 8.2: Added support for DNF types.
+
+### Fixed
+
+* Support `readonly` as a function name.
+* Added `__serialize` and `__unserialize` to magic method list.
+* Fixed bounds check in `Name::slice()`.
+* Fixed formatting preservation when adding attributes to a class/method/etc that previously had none.
+
+Version 4.14.0 (2022-05-31)
+---------------------------
+
+### Added
+
+* Added support for readonly classes.
+* Added `rawValue` attribute to `LNumber`, `DNumber` and `String_` nodes, which stores the unparsed
+  value of the literal (e.g. `"1_000"` rather than `1000`).
+
+Version 4.13.2 (2021-11-30)
+---------------------------
+
+### Added
+
+* Added builders for enums and enum cases.
+
+### Fixed
+
+* NullsafeMethodCall now extends from CallLike.
+* The `namespacedName` property populated by the `NameResolver` is now declared on relevant nodes,
+  to avoid a dynamic property deprecation warning with PHP 8.2.
+
+Version 4.13.1 (2021-11-03)
+---------------------------
+
+### Fixed
+
+* Support reserved keywords as enum cases.
+* Support array unpacking in constant expression evaluator.
+
+Version 4.13.0 (2021-09-20)
+---------------------------
+
+### Added
+
+* [PHP 8.1] Added support for intersection types using a new `IntersectionType` node. Additionally
+  a `ComplexType` parent class for `NullableType`, `UnionType` and `IntersectionType` has been
+  added.
+* [PHP 8.1] Added support for explicit octal literals.
+* [PHP 8.1] Added support for first-class callables. These are represented using a call whose first
+  argument is a `VariadicPlaceholder`. The representation is intended to be forward-compatible with
+  partial function application, just like the PHP feature itself. Call nodes now extend from
+  `Expr\CallLike`, which provides an `isFirstClassCallable()` method to determine whether a
+  placeholder id present. `getArgs()` can be used to assert that the call is not a first-class
+  callable and returns `Arg[]` rather than `array<Arg|VariadicPlaceholder>`.
+
+### Fixed
+
+* Multiple modifiers for promoted properties are now accepted. In particular this allows something
+  like `public readonly` for promoted properties.
+* Formatting-preserving pretty printing for comments in array literals has been fixed.
+
+Version 4.12.0 (2021-07-21)
+---------------------------
+
+### Added
+
+* [PHP 8.1] Added support for readonly properties (through a new `MODIFIER_READONLY`).
+* [PHP 8.1] Added support for final class constants.
+
+### Fixed
+
+* Fixed compatibility with PHP 8.1. `&` tokens are now canonicalized to the
+  `T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG` and `T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG` tokens
+  used in PHP 8.1. This happens unconditionally, regardless of whether the emulative lexer is used.
+
+Version 4.11.0 (2021-07-03)
+---------------------------
+
+### Added
+
+* `BuilderFactory::args()` now accepts named arguments.
+* `BuilderFactory::attribute()` has been added.
+* An `addAttribute()` method accepting an `Attribute` or `AttributeGroup` has been adde to all
+  builders that accept attributes, such as `Builder\Class_`.
+
+### Fixed
+
+* `NameResolver` now handles enums.
+* `PrettyPrinter` now prints backing enum type.
+* Builder methods for types now property handle `never` type.
+
+Version 4.10.5 (2021-05-03)
+---------------------------
+
+### Added
+
+* [PHP 8.1] Added support for enums. These are represented using the `Stmt\Enum_` and
+  `Stmt\EnumCase` nodes.
+* [PHP 8.1] Added support for never type. This type will now be returned as an `Identifier` rather
+  than `Name`.
+* Added `ClassConst` builder.
+
+### Changed
+
+* Non-UTF-8 code units in strings will now be hex-encoded.
+
+### Fixed
+
+* Fixed precedence of arrow functions.
+
+Version 4.10.4 (2020-12-20)
+---------------------------
+
+### Fixed
+
+* Fixed position information for variable-variables (#741).
+* Fixed position information for traits/interfaces preceded by if statement (#738).
+
+Version 4.10.3 (2020-12-03)
+---------------------------
+
+### Fixed
+
+* Fixed formatting-preserving pretty printing for `"{$x}"`.
+* Ternary expressions are now treated as non-associative in the pretty printer, in order to
+  generate code that is compatible with the parentheses requirement introduced in PHP 8.
+* Removed no longer necessary `error_clear_last()` call in lexer, which may interfere with fatal
+  error handlers if invoked during shutdown.
+
+
+Version 4.10.2 (2020-09-26)
+------------------
+
+### Fixed
+
+* Fixed check for token emulation conflicts with other libraries.
+
+Version 4.10.1 (2020-09-23)
+---------------------------
+
+### Added
+
+* Added support for recovering from a missing semicolon after a property or class constant
+  declaration.
+
+### Fixed
+
+* Fix spurious whitespace in formatting-preserving pretty printer when both removing and adding
+  elements at the start of a list.
+* Fix incorrect case-sensitivity in keyword token emulation.
+
+Version 4.10.0 (2020-09-19)
+---------------------------
+
+### Added
+
+* [PHP 8.0] Added support for attributes. These are represented using a new `AttributeGroup` node
+  containing `Attribute` nodes. A new `attrGroups` subnode is available on all node types that
+  support attributes, i.e. `Stmt\Class_`, `Stmt\Trait_`, `Stmt\Interface_`, `Stmt\Function_`,
+  `Stmt\ClassMethod`, `Stmt\ClassConst`, `Stmt\Property`, `Expr\Closure`, `Expr\ArrowFunction` and
+  `Param`.
+* [PHP 8.0] Added support for nullsafe properties inside interpolated strings, in line with an
+  upstream change.
+
+### Fixed
+
+* Improved compatibility with other libraries that use forward compatibility defines for PHP tokens.
+
+Version 4.9.1 (2020-08-30)
+--------------------------
+
+### Added
+
+* Added support for removing the first element of a list to the formatting-preserving pretty
+  printer.
+
+### Fixed
+
+* Allow member modifiers as part of namespaced names. These were missed when support for other
+  keywords was added.
+
+Version 4.9.0 (2020-08-18)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for named arguments, represented using a new `name` subnode on `Arg`.
+* [PHP 8.0] Added support for static return type, represented like a normal class return type.
+* [PHP 8.0] Added support for throw expression, represented using a new `Expr\Throw_` node. For
+  backwards compatibility reasons, throw expressions in statement context continue to be
+  represented using `Stmt\Throw_`.
+* [PHP 8.0] Added support for keywords as parts of namespaced names.
+
+### Fixed
+
+* Emit parentheses for class constant fetch with complex left-hand-side.
+* Emit parentheses for new/instanceof on complex class expression.
+
+Version 4.8.0 (2020-08-09)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for nullsafe operator, represented using the new
+  `Expr\NullsafePropertyFetch` and `Expr\NullsafeMethodCall` nodes.
+* Added `phpVersion` option to the emulative lexer, which allows controlling the target version to
+  emulate (defaults to the latest available, currently PHP 8.0). This is useful to parse code that
+  uses reserved keywords from newer PHP versions as identifiers.
+
+Version 4.7.0 (2020-07-25)
+--------------------------
+
+### Added
+
+* Add `ParentConnectingVisitor` and `NodeConnectingVisitor` classes.
+* [PHP 8.0] Added support for match expressions. These are represented using a new `Expr\Match_`
+  containing `MatchArm`s.
+* [PHP 8.0] Added support for trailing comma in closure use lists.
+
+### Fixed
+
+* Fixed missing error for unterminated comment with trailing newline (#688).
+* Compatibility with PHP 8.0 has been restored: Namespaced names are now always represented by
+  `T_NAME_*` tokens, using emulationg on older PHP versions. Full support for reserved keywords
+  in namespaced names is not yet present.
+
+Version 4.6.0 (2020-07-02)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for trailing commas in parameter lists.
+* [PHP 8.0] Added support for constructor promotion. The parameter visibility is stored in
+  `Node\Param::$flags`.
+
+### Fixed
+
+* Comment tokens now always follow the PHP 8 interpretation, and do not include trailing
+  whitespace.
+* As a result of the previous change, some whitespace issues when inserting a statement into a
+  method containing only a comment, and using the formatting-preserving pretty printer, have been
+  resolved.
+
+Version 4.5.0 (2020-06-03)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for the mixed type. This means `mixed` types are now parsed as an
+  `Identifier` rather than a `Name`.
+* [PHP 8.0] Added support for catching without capturing the exception. This means that
+  `Catch_::$var` may now be null.
+
+Version 4.4.0 (2020-04-10)
+--------------------------
+
+### Added
+
+* Added support for passing union types in builders.
+* Added end line, token position and file position information for comments.
+* Added `getProperty()` method to `ClassLike` nodes.
+
+### Fixed
+
+* Fixed generation of invalid code when using the formatting preserving pretty printer, and
+  inserting code next to certain nop statements. The formatting is still ugly though.
+* `getDocComment()` no longer requires that the very last comment before a node be a doc comment.
+  There may not be non-doc comments between the doc comment and the declaration.
+* Allowed arbitrary expressions in `isset()` and `list()`, rather than just variables.
+  In particular, this allows `isset(($x))`, which is legal PHP code.
+* [PHP 8.0] Add support for [variable syntax tweaks RFC](https://wiki.php.net/rfc/variable_syntax_tweaks).
+
+Version 4.3.0 (2019-11-08)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for union types using a new `UnionType` node.
+
+Version 4.2.5 (2019-10-25)
+--------------------------
+
+### Changed
+
+* Tests and documentation are no longer included in source archives. They can still be accessed
+  by cloning the repository.
+* php-yacc is now used to generate the parser. This has no impact on users of the library.
+
+Version 4.2.4 (2019-09-01)
+--------------------------
+
+### Added
+
+* Added getProperties(), getConstants() and getTraitUses() to ClassLike. (#629, #630)
+
+### Fixed
+
+* Fixed flexible heredoc emulation to check for digits after the end label. This synchronizes
+  behavior with the upcoming PHP 7.3.10 release.
+
+Version 4.2.3 (2019-08-12)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for numeric literal separators. (#615)
+
+### Fixed
+
+* Fixed resolution of return types for arrow functions. (#613)
+* Fixed compatibility with PHP 7.4.
+
+Version 4.2.2 (2019-05-25)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for arrow functions using a new `Expr\ArrowFunction` node. (#602)
+* [PHP 7.4] Add support for array spreads, using a new `unpack` subnode on `ArrayItem`. (#609)
+* Added support for inserting into empty list nodes in the formatting preserving pretty printer.
+
+### Changed
+
+* `php-parse` will now print messages to stderr, so that stdout only contains the actual result of
+  the operation (such as a JSON dump). (#605)
+
+### Fixed
+
+* Fixed attribute assignment for zero-length nop statements, and a related assertion failure in
+  the formatting-preserving pretty printer. (#589)
+
+Version 4.2.1 (2019-02-16)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for `??=` operator through a new `AssignOp\Coalesce` node. (#575)
+
 Version 4.2.0 (2019-01-12)
 --------------------------
 

CONTRIBUTING.md

@@ -0,0 +1,4 @@
+## Coding Style
+
+This project uses PSR-12 with consistent brace placement. This means that the opening brace is
+always on the same line, even for class and method declarations.

LICENSE

@@ -1,31 +1,29 @@
-Copyright (c) 2011-2018 by Nikita Popov.
+BSD 3-Clause License
 
-Some rights reserved.
+Copyright (c) 2011, Nikita Popov
+All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
+modification, are permitted provided that the following conditions are met:
-met:
 
-    * Redistributions of source code must retain the above copyright
+1. Redistributions of source code must retain the above copyright notice, this
-      notice, this list of conditions and the following disclaimer.
+   list of conditions and the following disclaimer.
 
-    * Redistributions in binary form must reproduce the above
+2. Redistributions in binary form must reproduce the above copyright notice,
-      copyright notice, this list of conditions and the following
+   this list of conditions and the following disclaimer in the documentation
-      disclaimer in the documentation and/or other materials provided
+   and/or other materials provided with the distribution.
-      with the distribution.
 
-    * The names of the contributors may not be used to endorse or
+3. Neither the name of the copyright holder nor the names of its
-      promote products derived from this software without specific
+   contributors may be used to endorse or promote products derived from
-      prior written permission.
+   this software without specific prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1,12 +1,14 @@
 PHP Parser
 ==========
 
-[![Build Status](https://travis-ci.org/nikic/PHP-Parser.svg?branch=master)](https://travis-ci.org/nikic/PHP-Parser) [![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
+[![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
 
-This is a PHP 5.2 to PHP 7.3 parser written in PHP. Its purpose is to simplify static code analysis and
+This is a PHP parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
-[**Documentation for version 4.x**][doc_master] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 7.3).
+[Documentation for version 5.x][doc_master] (in development; for running on PHP >= 7.1; for parsing PHP 7.0 to PHP 8.2, with limited support for parsing PHP 5.x).
+
+[**Documentation for version 4.x**][doc_4_x] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 8.2).
 
 [Documentation for version 3.x][doc_3_x] (unsupported; for running on PHP >= 5.5; for parsing PHP 5.2 to PHP 7.2).
 
@@ -15,12 +17,12 @@ Features
 
 The main features provided by this library are:
 
- * Parsing PHP 5 and PHP 7 code into an abstract syntax tree (AST).
+ * Parsing PHP 7, and PHP 8 code into an abstract syntax tree (AST).
    * Invalid code can be parsed into a partial AST.
    * The AST contains accurate location information.
  * Dumping the AST in human-readable form.
  * Converting an AST back to PHP code.
-   * Experimental: Formatting can be preserved for partially changed ASTs.
+   * Formatting can be preserved for partially changed ASTs.
  * Infrastructure to traverse and modify ASTs.
  * Resolution of namespaced names.
  * Evaluation of constant expressions.
@@ -51,7 +53,7 @@ function test($foo)
 }
 CODE;
 
-$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+$parser = (new ParserFactory())->createForNewestSupportedVersion();
 try {
     $ast = $parser->parse($code);
 } catch (Error $error) {
@@ -215,11 +217,12 @@ Component documentation:
  * [JSON representation](doc/component/JSON_representation.markdown)
    * JSON encoding and decoding of ASTs
  * [Performance](doc/component/Performance.markdown)
-   * Disabling XDebug
+   * Disabling Xdebug
    * Reusing objects
    * Garbage collection impact
  * [Frequently asked questions](doc/component/FAQ.markdown)
    * Parent and sibling references
 
  [doc_3_x]: https://github.com/nikic/PHP-Parser/tree/3.x/doc
+ [doc_4_x]: https://github.com/nikic/PHP-Parser/tree/4.x/doc
  [doc_master]: https://github.com/nikic/PHP-Parser/tree/master/doc

UPGRADE-5.0.md

@@ -0,0 +1,188 @@
+Upgrading from PHP-Parser 4.x to 5.0
+====================================
+
+### PHP version requirements
+
+PHP-Parser now requires PHP 7.1 or newer to run. It is however still possible to *parse* code for older versions, while running on a newer version.
+
+### PHP 5 parsing support
+
+The dedicated parser for PHP 5 has been removed. The PHP 7 parser now accepts a `PhpVersion` argument, which can be used to improve compatibility with older PHP versions.
+
+In particular, if an older `PhpVersion` is specified, then:
+
+ * For versions before PHP 7.0, `$foo =& new Bar()` assignments are allowed without error.
+ * For versions before PHP 7.0, invalid octal literals `089` are allowed without error.
+ * Type hints are interpreted as a class `Name` or as a built-in `Identifier` depending on PHP
+   version, for example `int` is treated as a class name on PHP 5.6 and as a built-in on PHP 7.0.
+
+However, some aspects of PHP 5 parsing are no longer supported:
+
+ * Some variables like `$$foo[0]` are valid in both PHP 5 and PHP 7, but have different interpretation. In that case, the PHP 7 AST will always be constructed (`($$foo)[0]` rather than `${$foo[0]}`).
+ * Declarations of the form `global $$var[0]` are not supported in PHP 7 and will cause a parse error. In error recovery mode, it is possible to continue parsing after such declarations.
+ * The PHP 7 parser will accept many constructs that are not valid in PHP 5. However, this was also true of the dedicated PHP 5 parser.
+
+The following symbols are affected by this removal:
+
+ * The `PhpParser\Parser\Php5` class has been removed.
+ * The `PhpParser\Parser\Multiple` class has been removed. While not strictly related to PHP 5 support, this functionality is no longer useful without it.
+ * The `PhpParser\ParserFactory::ONLY_PHP5` and `PREFER_PHP5` options have been removed.
+ * The `PhpParser\ParserFactory::PREFER_PHP7` option is now equivalent to `ONLY_PHP7`.
+
+### Changes to the parser factory
+
+The `ParserFactory::create()` method is deprecated in favor of three new methods that provide more fine-grained control over the PHP version being targeted:
+
+ * `createForNewestSupportedVersion()`: Use this if you don't know the PHP version of the code you're parsing. It's better to assume a too new version than a too old one.
+ * `createForHostVersion()`: Use this if you're parsing code for the PHP version you're running on.
+ * `createForVersion()`: Use this if you know the PHP version of the code you want to parse.
+
+In all cases, the PHP version is a fairly weak hint that is only used on a best-effort basis. The parser will usually accept code for newer versions if it does not have any backwards-compatibility implications.
+
+For example, if you specify version `"8.0"`, then `class ReadOnly {}` is treated as a valid class declaration, while using `public readonly int $prop` will lead to a parse error. However, `final public const X = Y;` will be accepted in both cases.
+
+```php
+use PhpParser\ParserFactory;
+use PhpParser\PhpVersion;
+
+$factory = new ParserFactory;
+
+# Before
+$parser = $factory->create(ParserFactory::PREFER_PHP7);
+
+# After (this is roughly equivalent to PREFER_PHP7 behavior)
+$parser = $factory->createForNewestSupportedVersion();
+# Or
+$parser = $factory->createForHostVersion();
+
+# Before
+$parser = $factory->create(ParserFactory::ONLY_PHP5);
+# After (supported on a best-effort basis)
+$parser = $factory->createForVersion(PhpVersion::fromString("5.6"));
+```
+
+### Changes to the array destructuring representation
+
+Previously, the `list($x) = $y` destructuring syntax was represented using a `Node\Expr\List_`
+node, while `[$x] = $y` used a `Node\Expr\Array_` node, the same used for the creation (rather than
+destructuring) of arrays.
+
+Now, destructuring is always represented using `Node\Expr\List_`. The `kind` attribute with value
+`Node\Expr\List_::KIND_LIST` or `Node\Expr\List_::KIND_ARRAY` specifies which syntax was actually
+used.
+
+### Renamed nodes
+
+A number of AST nodes have been renamed or moved in the AST hierarchy:
+
+ * `Node\Scalar\LNumber` is now `Node\Scalar\Int_`.
+ * `Node\Scalar\DNumber` is now `Node\Scalar\Float_`.
+ * `Node\Scalar\Encapsed` is now `Node\Scalar\InterpolatedString`.
+ * `Node\Scalar\EncapsedStringPart` is now `Node\InterpolatedStringPart` and no longer extends
+   `Node\Scalar` or `Node\Expr`.
+ * `Node\Expr\ArrayItem` is now `Node\ArrayItem` and no longer extends `Node\Expr`.
+ * `Node\Expr\ClosureUse` is now `Node\ClosureUse` and no longer extends `Node\Expr`.
+ * `Node\Stmt\DeclareDeclare` is now `Node\DeclareItem` and no longer extends `Node\Stmt`.
+ * `Node\Stmt\PropertyProperty` is now `Node\PropertyItem` and no longer extends `Node\Stmt`.
+ * `Node\Stmt\StaticVar` is now `Node\StaticVar` and no longer extends `Node\Stmt`.
+ * `Node\Stmt\UseUse` is now `Node\UseItem` and no longer extends `Node\Stmt`.
+
+The old class names have been retained as aliases for backwards compatibility. However, the `Node::getType()` method will now always return the new name (e.g. `ClosureUse` instead of `Expr_ClosureUse`).
+
+### Modifiers
+
+Modifier flags (as used by the `$flags` subnode of `Class_`, `ClassMethod`, `Property`, etc.) are now available as class constants on a separate `PhpParser\Modifiers` class, instead of being part of `PhpParser\Node\Stmt\Class_`, to make it clearer that these are used by many different nodes. The old constants are deprecated, but are still available.
+
+```
+PhpParser\Node\Stmt\Class_::MODIFIER_PUBLIC    -> PhpParser\Modifiers::PUBLIC
+PhpParser\Node\Stmt\Class_::MODIFIER_PROTECTED -> PhpParser\Modifiers::PROTECTED
+PhpParser\Node\Stmt\Class_::MODIFIER_PRIVATE   -> PhpParser\Modifiers::PRIVATE
+PhpParser\Node\Stmt\Class_::MODIFIER_STATIC    -> PhpParser\Modifiers::STATIC
+PhpParser\Node\Stmt\Class_::MODIFIER_ABSTRACT  -> PhpParser\Modifiers::ABSTRACT
+PhpParser\Node\Stmt\Class_::MODIFIER_FINAL     -> PhpParser\Modifiers::FINAL
+PhpParser\Node\Stmt\Class_::MODIFIER_READONLY  -> PhpParser\Modifiers::READONLY
+PhpParser\Node\Stmt\Class_::VISIBILITY_MODIFIER_MASK -> PhpParser\Modifiers::VISIBILITY_MASK
+```
+
+### Changes to the default pretty printer
+
+A number of changes to the standard pretty printer have been made, to make it match contemporary coding style conventions (and in particular PSR-12). Options to restore the previous behavior are not provided, but it is possible to override the formatting methods (such as `pStmt_ClassMethod`) with your preferred formatting.
+
+Return types are now formatted without a space before the `:`:
+
+```php
+# Before
+function test() : Type
+{
+}
+
+# After
+function test(): Type
+{
+}
+```
+
+`abstract` and `final` are now printed before visibility modifiers:
+
+```php
+# Before
+public abstract function test();
+
+# After
+abstract public function test();
+```
+
+A space is now printed between `use` and the following `(` for closures:
+
+```php
+# Before
+function () use($var) {
+};
+
+# After
+function () use ($var) {
+};
+```
+
+Backslashes in single-quoted strings are now only printed if they are necessary:
+
+```php
+# Before
+'Foo\\Bar';
+'\\\\';
+
+# After
+'Foo\Bar';
+'\\\\';
+```
+
+The pretty printer now accepts a `phpVersion` option, which accepts a `PhpVersion` object and defaults to PHP 7.0. The pretty printer will make formatting choices to make the code valid for that version. It currently controls the following behavior:
+
+* For PHP >= 7.0 (default), short array syntax `[]` will be used by default. This does not affect nodes that specify an explicit array syntax using the `kind` attribute.
+* For PHP >= 7.1, the short array syntax `[]` will be used for destructuring by default (instead of
+  `list()`). This does not affect nodes that specify and explicit syntax using the `kind` attribute.
+* For PHP >= 7.3, a newline is no longer forced after heredoc/nowdoc strings, as the requirement for this has been removed with the introduction of flexible heredoc/nowdoc strings.
+
+### Changes to token representation
+
+Tokens are now internally represented using the `PhpParser\Token` class, which exposes the same base interface as
+the `PhpToken` class introduced in PHP 8.0. On PHP 8.0 or newer, `PhpParser\Token` extends from `PhpToken`, otherwise
+it extends from a polyfill implementation. The most important parts of the interface may be summarized as follows:
+
+```php
+class Token {
+    public int $id;
+    public string $text;
+    public int $line;
+    public int $pos;
+
+    public function is(int|string|array $kind): bool;
+}
+```
+
+The `Lexer::getTokens()` method will now return an array of `Token`s, rather than an array of arrays and strings.
+Additionally, the token array is now terminated by a sentinel token with ID 0.
+
+### Other removed functionality
+
+ * The deprecated `Builder\Param::setTypeHint()` method has been removed in favor of `Builder\Param::setType()`.

bin/php-parse

@@ -10,7 +10,7 @@ foreach ([__DIR__ . '/../../../autoload.php', __DIR__ . '/../vendor/autoload.php
 
 ini_set('xdebug.max_nesting_level', 3000);
 
-// Disable XDebug var_dump() output truncation
+// Disable Xdebug var_dump() output truncation
 ini_set('xdebug.var_display_max_children', -1);
 ini_set('xdebug.var_display_max_data', -1);
 ini_set('xdebug.var_display_max_depth', -1);
@@ -26,13 +26,10 @@ if (empty($files)) {
     showHelp("Must specify at least one file.");
 }
 
-$lexer = new PhpParser\Lexer\Emulative(['usedAttributes' => [
+$lexerOptions = ['usedAttributes' => [
     'startLine', 'endLine', 'startFilePos', 'endFilePos', 'comments'
-]]);
+]];
-$parser = (new PhpParser\ParserFactory)->create(
+$parser = (new PhpParser\ParserFactory())->createForVersion($attributes['version'], $lexerOptions);
-    PhpParser\ParserFactory::PREFER_PHP7,
-    $lexer
-);
 $dumper = new PhpParser\NodeDumper([
     'dumpComments' => true,
     'dumpPositions' => $attributes['with-positions'],
@@ -45,14 +42,15 @@ $traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);
 foreach ($files as $file) {
     if (strpos($file, '<?php') === 0) {
         $code = $file;
-        echo "====> Code $code\n";
+        fwrite(STDERR, "====> Code $code\n");
     } else {
         if (!file_exists($file)) {
-            die("File $file does not exist.\n");
+            fwrite(STDERR, "File $file does not exist.\n");
+            exit(1);
         }
 
         $code = file_get_contents($file);
-        echo "====> File $file:\n";
+        fwrite(STDERR, "====> File $file:\n");
     }
 
     if ($attributes['with-recovery']) {
@@ -60,7 +58,7 @@ foreach ($files as $file) {
         $stmts = $parser->parse($code, $errorHandler);
         foreach ($errorHandler->getErrors() as $error) {
             $message = formatErrorMessage($error, $code, $attributes['with-column-info']);
-            echo $message . "\n";
+            fwrite(STDERR, $message . "\n");
         }
         if (null === $stmts) {
             continue;
@@ -70,25 +68,26 @@ foreach ($files as $file) {
             $stmts = $parser->parse($code);
         } catch (PhpParser\Error $error) {
             $message = formatErrorMessage($error, $code, $attributes['with-column-info']);
-            die($message . "\n");
+            fwrite(STDERR, $message . "\n");
+            exit(1);
         }
     }
 
     foreach ($operations as $operation) {
         if ('dump' === $operation) {
-            echo "==> Node dump:\n";
+            fwrite(STDERR, "==> Node dump:\n");
             echo $dumper->dump($stmts, $code), "\n";
         } elseif ('pretty-print' === $operation) {
-            echo "==> Pretty print:\n";
+            fwrite(STDERR, "==> Pretty print:\n");
             echo $prettyPrinter->prettyPrintFile($stmts), "\n";
         } elseif ('json-dump' === $operation) {
-            echo "==> JSON dump:\n";
+            fwrite(STDERR, "==> JSON dump:\n");
             echo json_encode($stmts, JSON_PRETTY_PRINT), "\n";
         } elseif ('var-dump' === $operation) {
-            echo "==> var_dump():\n";
+            fwrite(STDERR, "==> var_dump():\n");
             var_dump($stmts);
         } elseif ('resolve-names' === $operation) {
-            echo "==> Resolved names.\n";
+            fwrite(STDERR, "==> Resolved names.\n");
             $stmts = $traverser->traverse($stmts);
         }
     }
@@ -104,9 +103,9 @@ function formatErrorMessage(PhpParser\Error $e, $code, $withColumnInfo) {
 
 function showHelp($error = '') {
     if ($error) {
-        echo $error . "\n\n";
+        fwrite(STDERR, $error . "\n\n");
     }
-    die(<<<OUTPUT
+    fwrite($error ? STDERR : STDOUT, <<<'OUTPUT'
 Usage: php-parse [operations] file1.php [file2.php ...]
    or: php-parse [operations] "<?php code"
 Turn PHP source code into an abstract syntax tree.
@@ -121,6 +120,7 @@ Operations is a list of the following options (--dump by default):
     -c, --with-column-info  Show column-numbers for errors (if available)
     -P, --with-positions    Show positions in node dumps
     -r, --with-recovery     Use parsing with error recovery
+        --version=VERSION   Target specific PHP version (default: newest)
     -h, --help              Display this page
 
 Example:
@@ -131,6 +131,7 @@ Example:
 
 OUTPUT
     );
+    exit($error ? 1 : 0);
 }
 
 function parseArgs($args) {
@@ -140,6 +141,7 @@ function parseArgs($args) {
         'with-column-info' => false,
         'with-positions' => false,
         'with-recovery' => false,
+        'version' => PhpParser\PhpVersion::getNewestSupported(),
     ];
 
     array_shift($args);
@@ -190,7 +192,9 @@ function parseArgs($args) {
                 $parseOptions = false;
                 break;
             default:
-                if ($arg[0] === '-') {
+                if (preg_match('/^--version=(.*)$/', $arg, $matches)) {
+                    $attributes['version'] = PhpParser\PhpVersion::fromString($matches[1]);
+                } elseif ($arg[0] === '-') {
                     showHelp("Invalid operation $arg.");
                 } else {
                     $files[] = $arg;

composer.json

@@ -13,15 +13,18 @@
         }
     ],
     "require": {
-        "php": ">=7.0",
+        "php": ">=7.1",
-        "ext-tokenizer": "*"
+        "ext-tokenizer": "*",
+        "ext-json": "*",
+        "ext-ctype": "*"
     },
     "require-dev": {
-        "phpunit/phpunit": "^6.5 || ^7.0"
+        "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0",
+        "ircmaxell/php-yacc": "^0.0.7"
     },
     "extra": {
         "branch-alias": {
-            "dev-master": "4.2-dev"
+            "dev-master": "5.0-dev"
         }
     },
     "autoload": {
@@ -29,6 +32,11 @@
             "PhpParser\\": "lib/PhpParser"
         }
     },
+    "autoload-dev": {
+        "psr-4": {
+            "PhpParser\\": "test/PhpParser/"
+        }
+    },
     "bin": [
         "bin/php-parse"
     ]

doc/0_Introduction.markdown

@@ -1,7 +1,7 @@
 Introduction
 ============
 
-This project is a PHP 5.2 to PHP 7.3 parser **written in PHP itself**.
+This project is a PHP parser **written in PHP itself**.
 
 What is this for?
 -----------------
@@ -26,12 +26,29 @@ programmatic PHP code analysis are incidentally PHP developers, not C developers
 What can it parse?
 ------------------
 
-The parser supports parsing PHP 5.2-7.3.
+The parser supports parsing PHP 7 and PHP 8 code, with the following exceptions:
+
+ * Namespaced names containing whitespace (e.g. `Foo \ Bar` instead of `Foo\Bar`) are not supported.
+   These are illegal in PHP 8, but are legal in earlier versions. However, PHP-Parser does not
+   support them for any version.
+
+PHP-Parser 4.x had full support for parsing PHP 5. PHP-Parser 5.x has only limited support, with the
+following caveats:
+
+ * Some variable expressions like `$$foo[0]` are valid in both PHP 5 and PHP 7, but have different 
+   interpretation. In such cases, the PHP 7 AST will always be constructed (using `($$foo)[0]`
+   rather than `${$foo[0]}`).
+ * Declarations of the form `global $$var[0]` are not supported in PHP 7 and will cause a parse 
+   error. In error recovery mode, it is possible to continue parsing after such declarations.
 
 As the parser is based on the tokens returned by `token_get_all` (which is only able to lex the PHP
 version it runs on), additionally a wrapper for emulating tokens from newer versions is provided.
-This allows to parse PHP 7.3 source code running on PHP 7.0, for example. This emulation is somewhat
+This allows to parse PHP 8.0 source code running on PHP 7.1, for example. This emulation is not
-hacky and not perfect, but it should work well on any sane code.
+perfect, but works well in practice.
+
+Finally, it should be noted that the parser aims to accept all valid code, not reject all invalid
+code. It will generally accept code that is only valid in newer versions (even when targeting an
+older one), and accept code that is syntactically correct, but would result in a compiler error.
 
 What output does it produce?
 ----------------------------
@@ -56,10 +73,10 @@ array(
 ```
 
 This matches the structure of the code: An echo statement, which takes two strings as expressions,
-with the values `Hi` and `World!`.
+with the values `Hi` and `World`.
 
 You can also see that the AST does not contain any whitespace information (but most comments are saved).
-So using it for formatting analysis is not possible.
+However, it does retain accurate position information, which can be used to inspect precise formatting.
 
 What else can it do?
 --------------------
@@ -70,7 +87,7 @@ Apart from the parser itself this package also bundles support for some other, r
    that "pretty printing" does not imply that the output is especially pretty. It's just how it's
    called ;)
  * Support for serializing and unserializing the node tree to JSON
- * Support for dumping the node tree in a human readable form (see the section above for an
+ * Support for dumping the node tree in a human-readable form (see the section above for an
    example of how the output looks like)
  * Infrastructure for traversing and changing the AST (node traverser and node visitors)
  * A node visitor for resolving namespaced names

doc/2_Usage_of_basic_components.markdown

@@ -12,14 +12,14 @@ To bootstrap the library, include the autoloader generated by composer:
 require 'path/to/vendor/autoload.php';
 ```
 
-Additionally you may want to set the `xdebug.max_nesting_level` ini option to a higher value:
+Additionally, you may want to set the `xdebug.max_nesting_level` ini option to a higher value:
 
 ```php
 ini_set('xdebug.max_nesting_level', 3000);
 ```
 
 This ensures that there will be no errors when traversing highly nested node trees. However, it is
-preferable to disable XDebug completely, as it can easily make this library more than five times
+preferable to disable Xdebug completely, as it can easily make this library more than five times
 slower.
 
 Parsing
@@ -29,25 +29,29 @@ In order to parse code, you first have to create a parser instance:
 
 ```php
 use PhpParser\ParserFactory;
-$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+use PhpParser\PhpVersion;
+
+// Parser for the version you are running on.
+$parser = (new ParserFactory())->createForHostVersion();
+
+// Parser for the newest PHP version supported by the PHP-Parser library.
+$parser = (new ParserFactory())->createForNewestSupportedVersion();
+
+// Parser for a specific PHP version.
+$parser = (new ParserFactory())->createForVersion(PhpVersion::fromString('8.1'));
 ```
 
-The factory accepts a kind argument, that determines how different PHP versions are treated:
+Which version you should target depends on your use case. In many cases you will want to use the
+host version, as people typically analyze code for the version they are running on. However, when
+analyzing arbitrary code you are usually best off using the newest supported version, which tends
+to accept the widest range of code (unless there are breaking changes in PHP).
 
-Kind | Behavior
+The `createXYZ()` methods optionally accept an array of lexer options. Some use cases that require
------|---------
+customized lexer options are discussed in the [lexer documentation](component/Lexer.markdown).
-`ParserFactory::PREFER_PHP7` | Try to parse code as PHP 7. If this fails, try to parse it as PHP 5.
-`ParserFactory::PREFER_PHP5` | Try to parse code as PHP 5. If this fails, try to parse it as PHP 7.
-`ParserFactory::ONLY_PHP7` | Parse code as PHP 7.
-`ParserFactory::ONLY_PHP5` | Parse code as PHP 5.
 
-Unless you have a strong reason to use something else, `PREFER_PHP7` is a reasonable default.
+Subsequently, you can pass PHP code (including the opening `<?php` tag) to the `parse()` method in
-
+order to create a syntax tree. If a syntax error is encountered, an `PhpParser\Error` exception will
-The `create()` method optionally accepts a `Lexer` instance as the second argument. Some use cases
+be thrown by default:
-that require customized lexers are discussed in the [lexer documentation](component/Lexer.markdown).
-
-Subsequently you can pass PHP code (including the opening `<?php` tag) to the `parse` method in order to
-create a syntax tree. If a syntax error is encountered, an `PhpParser\Error` exception will be thrown:
 
 ```php
 <?php
@@ -62,7 +66,7 @@ function printLine($msg) {
 printLine('Hello World!!!');
 CODE;
 
-$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+$parser = (new ParserFactory())->createForHostVersion();
 
 try {
     $stmts = $parser->parse($code);
@@ -77,7 +81,7 @@ A parser instance can be reused to parse multiple files.
 Node dumping
 ------------
 
-To dump the abstact syntax tree in human readable form, a `NodeDumper` can be used:
+To dump the abstract syntax tree in human-readable form, a `NodeDumper` can be used:
 
 ```php
 <?php
@@ -171,7 +175,7 @@ with them easier they are grouped into three categories:
 
  * `PhpParser\Node\Stmt`s are statement nodes, i.e. language constructs that do not return
    a value and can not occur in an expression. For example a class definition is a statement.
-   It doesn't return a value and you can't write something like `func(class A {});`.
+   It doesn't return a value, and you can't write something like `func(class A {});`.
  * `PhpParser\Node\Expr`s are expression nodes, i.e. language constructs that return a value
    and thus can occur in other expressions. Examples of expressions are `$var`
    (`PhpParser\Node\Expr\Variable`) and `func()` (`PhpParser\Node\Expr\FuncCall`).
@@ -201,15 +205,14 @@ can then be retrieved using `hasAttribute()`, `getAttribute()` and `getAttribute
 By default the lexer adds the `startLine`, `endLine` and `comments` attributes. `comments` is an array
 of `PhpParser\Comment[\Doc]` instances.
 
-The start line can also be accessed using `getLine()`/`setLine()` (instead of `getAttribute('startLine')`).
+The start line can also be accessed using `getStartLine()` (instead of `getAttribute('startLine')`).
 The last doc comment from the `comments` attribute can be obtained using `getDocComment()`.
 
 Pretty printer
 --------------
 
-The pretty printer component compiles the AST back to PHP code. As the parser does not retain formatting
+The pretty printer component compiles the AST back to PHP code according to a specified scheme.
-information the formatting is done using a specified scheme. Currently there is only one scheme available,
+Currently, there is only one scheme available, namely `PhpParser\PrettyPrinter\Standard`.
-namely `PhpParser\PrettyPrinter\Standard`.
 
 ```php
 use PhpParser\Error;
@@ -218,8 +221,8 @@ use PhpParser\PrettyPrinter;
 
 $code = "<?php echo 'Hi ', hi\\getTarget();";
 
-$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+$parser = (new ParserFactory())->createForHostVersion();
-$prettyPrinter = new PrettyPrinter\Standard;
+$prettyPrinter = new PrettyPrinter\Standard();
 
 try {
     // parse
@@ -254,10 +257,13 @@ single expression using `prettyPrintExpr()`.
 The `prettyPrintFile()` method can be used to print an entire file. This will include the opening `<?php` tag
 and handle inline HTML as the first/last statement more gracefully.
 
+There is also a pretty-printing mode which retains formatting for parts of the AST that have not
+been changed, which requires additional setup.
+
 > Read more: [Pretty printing documentation](component/Pretty_printing.markdown)
 
-Node traversation
+Node traversal
------------------
+--------------
 
 The above pretty printing example used the fact that the source code was known and thus it was easy to
 write code that accesses a certain part of a node tree and changes it. Normally this is not the case.
@@ -272,7 +278,7 @@ use PhpParser\NodeTraverser;
 use PhpParser\ParserFactory;
 use PhpParser\PrettyPrinter;
 
-$parser        = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+$parser        = (new ParserFactory())->createForHostVersion();
 $traverser     = new NodeTraverser;
 $prettyPrinter = new PrettyPrinter\Standard;
 
@@ -303,8 +309,7 @@ The corresponding node visitor might look like this:
 use PhpParser\Node;
 use PhpParser\NodeVisitorAbstract;
 
-class MyNodeVisitor extends NodeVisitorAbstract
+class MyNodeVisitor extends NodeVisitorAbstract {
-{
     public function leaveNode(Node $node) {
         if ($node instanceof Node\Scalar\String_) {
             $node->value = 'foo';
@@ -326,7 +331,7 @@ public function afterTraverse(array $nodes);
 ```
 
 The `beforeTraverse()` method is called once before the traversal begins and is passed the nodes the
-traverser was called with. This method can be used for resetting values before traversation or
+traverser was called with. This method can be used for resetting values before traversal or
 preparing the tree for traversal.
 
 The `afterTraverse()` method is similar to the `beforeTraverse()` method, with the only difference that
@@ -342,8 +347,8 @@ The `enterNode()` method can additionally return the value `NodeTraverser::DONT_
 which instructs the traverser to skip all children of the current node. To furthermore prevent subsequent
 visitors from visiting the current node, `NodeTraverser::DONT_TRAVERSE_CURRENT_AND_CHILDREN` can be used instead.
 
-The `leaveNode()` method can additionally return the value `NodeTraverser::REMOVE_NODE`, in which
+Both methods can additionally return the value `NodeTraverser::REMOVE_NODE`, in which
-case the current node will be removed from the parent array. Furthermore it is possible to return
+case the current node will be removed from the parent array. Furthermore, it is possible to return
 an array of nodes, which will be merged into the parent array at the offset of the current node.
 I.e. if in `array(A, B, C)` the node `B` should be replaced with `array(X, Y, Z)` the result will
 be `array(A, X, Y, Z, C)`.
@@ -372,8 +377,9 @@ unqualified function and constant names. These are resolved at runtime and thus
 know which function they are referring to. In most cases this is a non-issue as the global functions
 are meant.
 
-Also the `NameResolver` adds a `namespacedName` subnode to class, function and constant declarations
+Additionally, the `NameResolver` adds a `namespacedName` subnode to class, function and constant
-that contains the namespaced name instead of only the shortname that is available via `name`.
+declarations that contains the namespaced name instead of only the shortname that is available via
+`name`.
 
 > Read more: [Name resolution documentation](component/Name_resolution.markdown)
 
@@ -396,7 +402,7 @@ use PhpParser\NodeVisitor\NameResolver;
 $inDir  = '/some/path';
 $outDir = '/some/other/path';
 
-$parser        = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+$parser        = (new ParserFactory())->createForNewestSupportedVersion();
 $traverser     = new NodeTraverser;
 $prettyPrinter = new PrettyPrinter\Standard;
 

doc/README.md

@@ -39,7 +39,7 @@ Component documentation
   * [JSON representation](component/JSON_representation.markdown)
     * JSON encoding and decoding of ASTs
   * [Performance](component/Performance.markdown)
-    * Disabling XDebug
+    * Disabling Xdebug
     * Reusing objects
     * Garbage collection impact
   * [Frequently asked questions](component/FAQ.markdown)

doc/component/AST_builders.markdown

@@ -101,7 +101,7 @@ abstract class SomeOtherClass extends SomeClass implements A\Few, \Interfaces
      *
      * @param SomeClass And takes a parameter
      */
-    public abstract function someMethod(SomeClass $someParam) : bool;
+    public abstract function someMethod(SomeClass $someParam): bool;
     protected function anotherMethod($someParam = 'test')
     {
         print $someParam;

doc/component/Constant_expression_evaluation.markdown

@@ -19,9 +19,9 @@ PHP-Parser supports evaluation of such constant expressions through the `ConstEx
 
 use PhpParser\{ConstExprEvaluator, ConstExprEvaluationException};
 
-$evalutator = new ConstExprEvaluator();
+$evaluator = new ConstExprEvaluator();
 try {
-    $value = $evalutator->evaluateSilently($someExpr);
+    $value = $evaluator->evaluateSilently($someExpr);
 } catch (ConstExprEvaluationException $e) {
     // Either the expression contains unsupported expression types,
     // or an error occurred during evaluation
@@ -45,7 +45,7 @@ use PhpParser\Node\{Expr, Scalar};
 $evaluator = new ConstExprEvaluator();
 
 // 10 / 0
-$expr = new Expr\BinaryOp\Div(new Scalar\LNumber(10), new Scalar\LNumber(0));
+$expr = new Expr\BinaryOp\Div(new Scalar\Int_(10), new Scalar\Int_(0));
 
 var_dump($evaluator->evaluateDirectly($expr)); // float(INF)
 // Warning: Division by zero
@@ -69,6 +69,7 @@ expressions, apart from the following:
  * `Scalar\MagicConst\*`
  * `Expr\ConstFetch` (only null/false/true are handled)
  * `Expr\ClassConstFetch`
+ * `Expr\New_` (since PHP 8.1)
 
 Handling these expression types requires non-local information, such as which global constants are
 defined. By default, the evaluator will throw a `ConstExprEvaluationException` when it encounters

doc/component/Error_handling.markdown

@@ -14,10 +14,10 @@ In order to receive information about not only the line, but also the column spa
 position attributes in the lexer need to be enabled:
 
 ```php
-$lexer = new PhpParser\Lexer(array(
+$lexerOptions = array(
     'usedAttributes' => array('comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'),
-));
+);
-$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::PREFER_PHP7, $lexer);
+$parser = (new PhpParser\ParserFactory())->createForHostVersion($lexerOptions);
 
 try {
     $stmts = $parser->parse($code);
@@ -36,7 +36,7 @@ if ($e->hasColumnInfo()) {
     echo $e->getRawMessage() . ' from ' . $e->getStartLine() . ':' . $e->getStartColumn($code)
         . ' to ' . $e->getEndLine() . ':' . $e->getEndColumn($code);
     // or:
-    echo $e->getMessageWithColumnInfo();
+    echo $e->getMessageWithColumnInfo($code);
 } else {
     echo $e->getMessage();
 }
@@ -56,7 +56,7 @@ To instead collect all encountered errors into an array, while trying to continu
 an instance of `ErrorHandler\Collecting` can be passed to the `Parser::parse()` method. A usage example:
 
 ```php
-$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::ONLY_PHP7);
+$parser = (new PhpParser\ParserFactory())->createForHostVersion();
 $errorHandler = new PhpParser\ErrorHandler\Collecting;
 
 $stmts = $parser->parse($code, $errorHandler);
@@ -72,4 +72,6 @@ if (null !== $stmts) {
 }
 ```
 
+The partial AST may contain `Expr\Error` nodes that indicate that an error occurred while parsing an expression.
+
 The `NameResolver` visitor also accepts an `ErrorHandler` as a constructor argument.

doc/component/FAQ.markdown

@@ -7,28 +7,21 @@ Frequently Asked Questions
 How can the parent of a node be obtained?
 -----
 
-The AST does not store parent nodes by default. However, it is easy to add a custom parent node
+The AST does not store parent nodes by default. However, the `ParentConnectingVisitor` can be used to achieve this:
-attribute using a custom node visitor:
 
 ```php
-use PhpParser\Node;
+use PhpParser\NodeTraverser;
-use PhpParser\NodeVisitorAbstract;
+use PhpParser\NodeVisitor\ParentConnectingVisitor;
+use PhpParser\ParserFactory;
 
-class ParentConnector extends NodeVisitorAbstract {
+$code = '...';
-    private $stack;
+
-    public function beforeTraverse(array $nodes) {
+$traverser = new NodeTraverser;
-        $this->stack = [];
+$traverser->addVisitor(new ParentConnectingVisitor);
-    }
+
-    public function enterNode(Node $node) {
+$parser = (new ParserFactory())->createForHostVersion();
-        if (!empty($this->stack)) {
+$ast    = $parser->parse($code);
-            $node->setAttribute('parent', $this->stack[count($this->stack)-1]);
+$ast    = $traverser->traverse($ast);
-        }
-        $this->stack[] = $node;
-    }
-    public function leaveNode(Node $node) {
-        array_pop($this->stack);
-    }
-}
 ```
 
 After running this visitor, the parent node can be obtained through `$node->getAttribute('parent')`.
@@ -36,33 +29,27 @@ After running this visitor, the parent node can be obtained through `$node->getA
 How can the next/previous sibling of a node be obtained?
 -----
 
-Again, siblings are not stored by default, but the visitor from the previous entry can be easily
+Again, siblings are not stored by default, but the `NodeConnectingVisitor` can be used to store
-extended to store the previous / next node with a common parent as well:
+the previous / next node with a common parent as well:
 
 ```php
-use PhpParser\Node;
+use PhpParser\NodeTraverser;
-use PhpParser\NodeVisitorAbstract;
+use PhpParser\NodeVisitor\NodeConnectingVisitor;
+use PhpParser\ParserFactory;
 
-class NodeConnector extends NodeVisitorAbstract {
+$code = '...';
-    private $stack;
+
-    private $prev;
+$traverser = new NodeTraverser;
-    public function beforeTraverse(array $nodes) {
+$traverser->addVisitor(new NodeConnectingVisitor);
-        $this->stack = [];
+
-        $this->prev = null;
+$parser = (new ParserFactory())->createForHostVersion();
-    }
+$ast    = $parser->parse($code);
-    public function enterNode(Node $node) {
+$ast    = $traverser->traverse($ast);
-        if (!empty($this->stack)) {
-            $node->setAttribute('parent', $this->stack[count($this->stack)-1]);
-        }
-        if ($this->prev && $this->prev->getAttribute('parent') == $node->getAttribute('parent')) {
-            $node->setAttribute('prev', $this->prev);
-            $this->prev->setAttribute('next', $node);
-        }
-        $this->stack[] = $node;
-    }
-    public function leaveNode(Node $node) {
-        $this->prev = $node;
-        array_pop($this->stack);
-    }
-}
 ```
+
+After running this visitor, the parent node can be obtained through `$node->getAttribute('parent')`,
+the previous node can be obtained through `$node->getAttribute('previous')`, and the next node can be
+obtained through `$node->getAttribute('next')`.
+
+`ParentConnectingVisitor` and `NodeConnectingVisitor` should not be used at the same time. The latter
+includes the functionality of the former.

doc/component/JSON_representation.markdown

@@ -18,7 +18,7 @@ function printLine($msg) {
 }
 CODE;
 
-$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+$parser = (new ParserFactory())->createForHostVersion();
 
 try {
     $stmts = $parser->parse($code);

doc/component/Lexer.markdown

@@ -42,16 +42,17 @@ The attributes used in this example match the default behavior of the lexer. The
 > **Note:** The example in this section is outdated in that this information is directly available in the AST: While
 > `$property->isPublic()` does not distinguish between `public` and `var`, directly checking `$property->flags` for
 > the `$property->flags & Class_::VISIBILITY_MODIFIER_MASK) === 0` allows making this distinction without resorting to
-> tokens. However the general idea behind the example still applies in other cases.
+> tokens. However, the general idea behind the example still applies in other cases.
 
 The token offset information is useful if you wish to examine the exact formatting used for a node. For example the AST
 does not distinguish whether a property was declared using `public` or using `var`, but you can retrieve this
 information based on the token position:
 
 ```php
+/** @param PhpParser\Token[] $tokens */
 function isDeclaredUsingVar(array $tokens, PhpParser\Node\Stmt\Property $prop) {
-    $i = $prop->getAttribute('startTokenPos');
+    $i = $prop->getStartTokenPos();
-    return $tokens[$i][0] === T_VAR;
+    return $tokens[$i]->id === T_VAR;
 }
 ```
 
@@ -72,12 +73,12 @@ class MyNodeVisitor extends PhpParser\NodeVisitorAbstract {
     }
 }
 
-$lexer = new PhpParser\Lexer(array(
+$lexerOptions = array(
     'usedAttributes' => array(
         'comments', 'startLine', 'endLine', 'startTokenPos', 'endTokenPos'
     )
-));
+);
-$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::ONLY_PHP7, $lexer);
+$parser = (new PhpParser\ParserFactory())->createForHostVersion($lexerOptions);
 
 $visitor = new MyNodeVisitor();
 $traverser = new PhpParser\NodeTraverser();
@@ -111,14 +112,15 @@ The `startLexing()` method is invoked whenever the `parse()` method of the parse
 code that is to be lexed (including the opening tag). It can be used to reset state or preprocess the source code or tokens. The
 passed `ErrorHandler` should be used to report lexing errors.
 
-The `getTokens()` method returns the current token array, in the usual `token_get_all()` format. This method is not
+The `getTokens()` method returns the current array of `PhpParser\Token`s, which are compatible with the PHP 8 `PhpToken`
-used by the parser (which uses `getNextToken()`), but is useful in combination with the token position attributes.
+class. This method is not used by the parser (which uses `getNextToken()`), but is useful in combination with the token
+position attributes.
 
 The `handleHaltCompiler()` method is called whenever a `T_HALT_COMPILER` token is encountered. It has to return the
 remaining string after the construct (not including `();`).
 
-The `getNextToken()` method returns the ID of the next token (as defined by the `Parser::T_*` constants). If no more
+The `getNextToken()` method returns the ID of the next token (in the sense of `Token::$id`). If no more
-tokens are available it must return `0`, which is the ID of the `EOF` token. Furthermore the string content of the
+tokens are available it must return `0`, which is the ID of the `EOF` token. Furthermore, the string content of the
 token should be written into the by-reference `$value` parameter (which will then be available as `$n` in the parser).
 
 ### Attribute handling
@@ -144,10 +146,10 @@ class KeepOriginalValueLexer extends Lexer // or Lexer\Emulative
     public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) {
         $tokenId = parent::getNextToken($value, $startAttributes, $endAttributes);
 
-        if ($tokenId == Tokens::T_CONSTANT_ENCAPSED_STRING   // non-interpolated string
+        if ($tokenId == \T_CONSTANT_ENCAPSED_STRING   // non-interpolated string
-            || $tokenId == Tokens::T_ENCAPSED_AND_WHITESPACE // interpolated string
+            || $tokenId == \T_ENCAPSED_AND_WHITESPACE // interpolated string
-            || $tokenId == Tokens::T_LNUMBER                 // integer
+            || $tokenId == \T_LNUMBER                 // integer
-            || $tokenId == Tokens::T_DNUMBER                 // floating point number
+            || $tokenId == \T_DNUMBER                 // floating point number
         ) {
             // could also use $startAttributes, doesn't really matter here
             $endAttributes['originalValue'] = $value;

doc/component/Name_resolution.markdown

@@ -10,7 +10,7 @@ visitor (NameResolver) based on it.
 The NameResolver visitor
 ------------------------
 
-The `NameResolver` visitor can (and for nearly all uses of the AST, is) be applied to resolve names
+The `NameResolver` visitor can (and for nearly all uses of the AST, should) be applied to resolve names
 to their fully-qualified form, to the degree that this is possible.
 
 ```php
@@ -53,7 +53,7 @@ name. Once again, if an unqualified function or constant name cannot be resolved
 `resolvedName` attribute will not be present, and instead a `namespacedName` attribute is added.
 
 The `replaceNodes` attribute is useful if you wish to perform modifications on the AST, as you
-probably do not wish the resoluting code to have fully resolved names as a side-effect.
+probably do not wish the resulting code to have fully resolved names as a side-effect.
 
 The NameContext
 ---------------

doc/component/Performance.markdown

@@ -8,20 +8,20 @@ described in the following.
 Xdebug
 ------
 
-Running PHP with XDebug adds a lot of overhead, especially for code that performs many method calls.
+Running PHP with Xdebug adds a lot of overhead, especially for code that performs many method calls.
-Just by loading XDebug (without enabling profiling or other more intrusive XDebug features), you
+Just by loading Xdebug (without enabling profiling or other more intrusive Xdebug features), you
 can expect that code using PHP-Parser will be approximately *five times slower*.
 
-As such, you should make sure that XDebug is not loaded when using this library. Note that setting
+As such, you should make sure that Xdebug is not loaded when using this library. Note that setting
-the `xdebug.default_enable=0` ini option does *not* disable XDebug. The *only* way to disable
+the `xdebug.default_enable=0` ini option does *not* disable Xdebug. The *only* way to disable
-XDebug is to not load the extension in the first place.
+Xdebug is to not load the extension in the first place.
 
-If you are building a command-line utility for use by developers (who often have XDebug enabled),
+If you are building a command-line utility for use by developers (who often have Xdebug enabled),
-you may want to consider automatically restarting PHP with XDebug unloaded. The
+you may want to consider automatically restarting PHP with Xdebug unloaded. The
 [composer/xdebug-handler](https://github.com/composer/xdebug-handler) package can be used to do
 this.
 
-If you do run with XDebug, you may need to increase the `xdebug.max_nesting_level` option to a
+If you do run with Xdebug, you may need to increase the `xdebug.max_nesting_level` option to a
 higher level, such as 3000. While the parser itself is recursion free, most other code working on
 the AST uses recursion and will generate an error if the value of this option is too low.
 

doc/component/Pretty_printing.markdown

@@ -31,10 +31,16 @@ expression.
 Customizing the formatting
 --------------------------
 
-Apart from an `shortArraySyntax` option, the default pretty printer does not provide any
+The pretty printer respects a number of `kind` attributes used by some notes (e.g., whether an
-functionality to customize the formatting of the generated code. The pretty printer does respect a
+integer should be printed as decimal, hexadecimal, etc). Additionally, it supports two options:
-number of `kind` attributes used by some notes (e.g., whether an integer should be printed as
+
-decimal, hexadecimal, etc), but there are no options to control brace placement or similar.
+* `phpVersion` (defaults to 7.0) allows opting into formatting that is not supported by older PHP
+  versions.
+* `shortArraySyntax` determines the used array syntax if the `kind` attribute is not set. This is
+  a legacy option, and `phpVersion` should be used to control this behavior instead.
+
+However, the default pretty printer does not provide any functionality for fine-grained
+customization of code formatting.
 
 If you want to make minor changes to the formatting, the easiest way is to extend the pretty printer
 and override the methods responsible for the node types you are interested in.
@@ -46,29 +52,27 @@ default pretty printer with an existing library for code reformatting, such as
 Formatting-preserving pretty printing
 -------------------------------------
 
-> **Note:** This functionality is **experimental** and not yet complete.
-
 For automated code refactoring, migration and similar, you will usually only want to modify a small
 portion of the code and leave the remainder alone. The basic pretty printer is not suitable for
 this, because it will also reformat parts of the code which have not been modified.
 
-Since PHP-Parser 4.0, an experimental formatting-preserving pretty-printing mode is available, which
+Since PHP-Parser 4.0, a formatting-preserving pretty-printing mode is available, which
 attempts to preserve the formatting of code (those AST nodes that have not changed) and only reformat
 code which has been modified or newly inserted.
 
 Use of the formatting-preservation functionality requires some additional preparatory steps:
 
 ```php
-use PhpParser\{Lexer, NodeTraverser, NodeVisitor, Parser, PrettyPrinter};
+use PhpParser\{Lexer, NodeTraverser, NodeVisitor, ParserFactory, PrettyPrinter};
 
-$lexer = new Lexer\Emulative([
+$lexerOptions = new [
     'usedAttributes' => [
         'comments',
         'startLine', 'endLine',
         'startTokenPos', 'endTokenPos',
     ],
-]);
+];
-$parser = new Parser\Php7($lexer);
+$parser = (new ParserFactory())->createForHostVersion($lexerOptions);
 
 $traverser = new NodeTraverser();
 $traverser->addVisitor(new NodeVisitor\CloningVisitor());
@@ -86,11 +90,9 @@ $newCode = $printer->printFormatPreserving($newStmts, $oldStmts, $oldTokens);
 ```
 
 If you make use of the name resolution functionality, you will likely want to disable the
-`replaceNodes` option. This will add resolved names as attributes, instead of directlying modifying
+`replaceNodes` option. This will add resolved names as attributes, instead of directly modifying
 the AST and causing spurious changes to the pretty printed code. For more information, see the
 [name resolution documentation](Name_resolution.markdown).
 
-This functionality is experimental and not yet fully implemented. It should not provide incorrect
+The formatting-preservation works on a best-effort basis and may sometimes reformat more code tha
-code, but it may sometimes reformat more code than necessary. Open issues are tracked in
+necessary. If you encounter problems while using this functionality, please open an issue.
-[issue #344](https://github.com/nikic/PHP-Parser/issues/344). If you encounter problems while using
-this functionality, please open an issue, so we know what to prioritize.

doc/component/Walking_the_AST.markdown

@@ -11,7 +11,7 @@ use PhpParser\{Node, NodeTraverser, NodeVisitorAbstract};
 $traverser = new NodeTraverser;
 $traverser->addVisitor(new class extends NodeVisitorAbstract {
     public function leaveNode(Node $node) {
-        if ($node instanceof Node\Scalar\LNumber) {
+        if ($node instanceof Node\Scalar\Int_) {
             return new Node\Scalar\String_((string) $node->value);
         }
     }
@@ -129,8 +129,7 @@ Now `$a && $b` will be replaced by `!($a && $b)`. Then the traverser will go int
 only) child of `!($a && $b)`, which is `$a && $b`. The transformation applies again and we end up
 with `!!($a && $b)`. This will continue until PHP hits the memory limit.
 
-Finally, two special replacement types are supported only by leaveNode. The first is removal of a
+Finally, there are two special replacement types. The first is removal of a node:
-node:
 
 ```php
 public function leaveNode(Node $node) {
@@ -165,8 +164,8 @@ This example will remove all calls to `var_dump()` which occur as expression sta
 that `var_dump($a);` will be removed, but `if (var_dump($a))` will not be removed (and there is no
 obvious way in which it can be removed).
 
-Next to removing nodes, it is also possible to replace one node with multiple nodes. Again, this
+Next to removing nodes, it is also possible to replace one node with multiple nodes. This
-only works inside leaveNode and only if the parent structure is an array.
+only works if the parent structure is an array.
 
 ```php
 public function leaveNode(Node $node) {
@@ -193,7 +192,7 @@ anonymous classes), you know that once you've seen a class declaration, there is
 checking all it's child nodes, because PHP does not allow nesting classes. In this case, you can
 instruct the traverser to not recurse into the class node:
 
-```
+```php
 private $classes = [];
 public function enterNode(Node $node) {
     if ($node instanceof Node\Stmt\Class_) {
@@ -211,7 +210,7 @@ after finding it. For example, if you are looking for the node of a class with a
 discounting exotic cases like conditionally defining a class two times), you can stop traversal
 once you found it:
 
-```
+```php
 private $class = null;
 public function enterNode(Node $node) {
     if ($node instanceof Node\Stmt\Class_ &&
@@ -251,7 +250,7 @@ Stmt_Return(
 
 the following method calls will be performed:
 
-```
+```php
 $visitorA->enterNode(Stmt_Return)
 $visitorB->enterNode(Stmt_Return)
 $visitorA->enterNode(Expr_Variable)
@@ -295,7 +294,7 @@ Simple node finding
 
 While the node visitor mechanism is very flexible, creating a node visitor can be overly cumbersome
 for minor tasks. For this reason a `NodeFinder` is provided, which can find AST nodes that either
-satisfy a certain callback, or which are instanced of a certain node type. A couple of examples are
+satisfy a certain callback, or which are instances of a certain node type. A couple of examples are
 shown in the following:
 
 ```php
@@ -312,7 +311,7 @@ $extendingClasses = $nodeFinder->find($stmts, function(Node $node) {
         && $node->extends !== null;
 });
 
-// Find first class occuring in the AST. Returns null if no class exists.
+// Find first class occurring in the AST. Returns null if no class exists.
 $class = $nodeFinder->findFirstInstanceOf($stmts, Node\Stmt\Class_::class);
 
 // Find first class that has name $name

grammar/README.md

@@ -1,11 +1,8 @@
 What do all those files mean?
 =============================
 
- * `php5.y`:             PHP 5 grammar written in a pseudo language
+ * `php.y`:              PHP 5-8 grammar written in a pseudo language
- * `php7.y`:             PHP 7 grammar written in a pseudo language
- * `tokens.y`:           Tokens definition shared between PHP 5 and PHP 7 grammars
  * `parser.template`:    A `kmyacc` parser prototype file for PHP
- * `tokens.template`:    A `kmyacc` prototype file for the `Tokens` class
  * `rebuildParsers.php`: Preprocesses the grammar and builds the parser using `kmyacc`
 
 .phpy pseudo language
@@ -21,8 +18,10 @@ applied to it:
 Building the parser
 ===================
 
-In order to rebuild the parser, you need [moriyoshi's fork of kmyacc](https://github.com/moriyoshi/kmyacc-forked).
+Run `php grammar/rebuildParsers.php` to rebuild the parsers. Additional options:
-After you compiled/installed it, run the `rebuildParsers.php` script.
 
-By default only the `Parser.php` is built. If you want to additionally emit debug symbols and create `y.output`, run the
+ * The `KMYACC` environment variable can be used to specify an alternative `kmyacc` binary.
-script with `--debug`. If you want to retain the preprocessed grammar pass `--keep-tmp-grammar`.
+   By default the `phpyacc` dev dependency will be used. To use the original `kmyacc`, you
+   need to compile [moriyoshi's fork](https://github.com/moriyoshi/kmyacc-forked).
+ * The `--debug` option enables emission of debug symbols and creates the `y.output` file.
+ * The `--keep-tmp-grammar` option preserves the preprocessed grammar file.

grammar/parser.template

@@ -1,4 +1,4 @@
-<?php
+<?php declare(strict_types=1);
 $meta #
 #semval($) $this->semValue
 #semval($,%t) $this->semValue
@@ -8,6 +8,7 @@ $meta #
 namespace PhpParser\Parser;
 
 use PhpParser\Error;
+use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Name;
@@ -17,12 +18,16 @@ use PhpParser\Node\Stmt;
 
 /* This is an automatically GENERATED file, which should not be manually edited.
  * Instead edit one of the following:
- *  * the grammar files grammar/php5.y or grammar/php7.y
+ *  * the grammar file grammar/php.y
  *  * the skeleton file grammar/parser.template
  *  * the preprocessing script grammar/rebuildParsers.php
  */
 class #(-p) extends \PhpParser\ParserAbstract
 {
+#tokenval
+    public const %s = %n;
+#endtokenval
+
     protected $tokenToSymbolMapSize = #(YYMAXLEX);
     protected $actionTableSize = #(YYLAST);
     protected $gotoTableSize = #(YYGLAST);

grammar/phpyLang.php

@@ -0,0 +1,184 @@
+<?php declare(strict_types=1);
+
+///////////////////////////////
+/// Utility regex constants ///
+///////////////////////////////
+
+const LIB = '(?(DEFINE)
+    (?<singleQuotedString>\'[^\\\\\']*+(?:\\\\.[^\\\\\']*+)*+\')
+    (?<doubleQuotedString>"[^\\\\"]*+(?:\\\\.[^\\\\"]*+)*+")
+    (?<string>(?&singleQuotedString)|(?&doubleQuotedString))
+    (?<comment>/\*[^*]*+(?:\*(?!/)[^*]*+)*+\*/)
+    (?<code>\{[^\'"/{}]*+(?:(?:(?&string)|(?&comment)|(?&code)|/)[^\'"/{}]*+)*+})
+)';
+
+const PARAMS = '\[(?<params>[^[\]]*+(?:\[(?&params)\][^[\]]*+)*+)\]';
+const ARGS   = '\((?<args>[^()]*+(?:\((?&args)\)[^()]*+)*+)\)';
+
+///////////////////////////////
+/// Preprocessing functions ///
+///////////////////////////////
+
+function preprocessGrammar($code) {
+    $code = resolveNodes($code);
+    $code = resolveMacros($code);
+    $code = resolveStackAccess($code);
+
+    return $code;
+}
+
+function resolveNodes($code) {
+    return preg_replace_callback(
+        '~\b(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
+        function ($matches) {
+            // recurse
+            $matches['params'] = resolveNodes($matches['params']);
+
+            $params = magicSplit(
+                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
+                $matches['params']
+            );
+
+            $paramCode = '';
+            foreach ($params as $param) {
+                $paramCode .= $param . ', ';
+            }
+
+            return 'new ' . $matches['name'] . '(' . $paramCode . 'attributes())';
+        },
+        $code
+    );
+}
+
+function resolveMacros($code) {
+    return preg_replace_callback(
+        '~\b(?<!::|->)(?!array\()(?<name>[a-z][A-Za-z]++)' . ARGS . '~',
+        function ($matches) {
+            // recurse
+            $matches['args'] = resolveMacros($matches['args']);
+
+            $name = $matches['name'];
+            $args = magicSplit(
+                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
+                $matches['args']
+            );
+
+            if ('attributes' === $name) {
+                assertArgs(0, $args, $name);
+                return '$this->startAttributeStack[#1] + $this->endAttributes';
+            }
+
+            if ('stackAttributes' === $name) {
+                assertArgs(1, $args, $name);
+                return '$this->startAttributeStack[' . $args[0] . ']'
+                       . ' + $this->endAttributeStack[' . $args[0] . ']';
+            }
+
+            if ('init' === $name) {
+                return '$$ = array(' . implode(', ', $args) . ')';
+            }
+
+            if ('push' === $name) {
+                assertArgs(2, $args, $name);
+
+                return $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0];
+            }
+
+            if ('pushNormalizing' === $name) {
+                assertArgs(2, $args, $name);
+
+                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
+                       . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
+            }
+
+            if ('toArray' == $name) {
+                assertArgs(1, $args, $name);
+
+                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
+            }
+
+            if ('parseVar' === $name) {
+                assertArgs(1, $args, $name);
+
+                return 'substr(' . $args[0] . ', 1)';
+            }
+
+            if ('parseEncapsed' === $name) {
+                assertArgs(3, $args, $name);
+
+                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\InterpolatedStringPart) {'
+                       . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, ' . $args[1] . ', ' . $args[2] . '); } }';
+            }
+
+            if ('makeNop' === $name) {
+                assertArgs(3, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                       . ' if (isset($startAttributes[\'comments\']))'
+                       . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
+                       . ' else { ' . $args[0] . ' = null; }';
+            }
+
+            if ('makeZeroLengthNop' == $name) {
+                assertArgs(2, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                       . ' if (isset($startAttributes[\'comments\']))'
+                       . ' { ' . $args[0] . ' = new Stmt\Nop($this->createCommentNopAttributes($startAttributes[\'comments\'])); }'
+                       . ' else { ' . $args[0] . ' = null; }';
+            }
+
+            if ('prependLeadingComments' === $name) {
+                assertArgs(1, $args, $name);
+
+                return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
+                       . 'if (!empty($attrs[\'comments\'])) {'
+                       . '$stmts[0]->setAttribute(\'comments\', '
+                       . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
+            }
+
+            return $matches[0];
+        },
+        $code
+    );
+}
+
+function assertArgs($num, $args, $name) {
+    if ($num != count($args)) {
+        die('Wrong argument count for ' . $name . '().');
+    }
+}
+
+function resolveStackAccess($code) {
+    $code = preg_replace('/\$\d+/', '$this->semStack[$0]', $code);
+    $code = preg_replace('/#(\d+)/', '$$1', $code);
+    return $code;
+}
+
+function removeTrailingWhitespace($code) {
+    $lines = explode("\n", $code);
+    $lines = array_map('rtrim', $lines);
+    return implode("\n", $lines);
+}
+
+//////////////////////////////
+/// Regex helper functions ///
+//////////////////////////////
+
+function regex($regex) {
+    return '~' . LIB . '(?:' . str_replace('~', '\~', $regex) . ')~';
+}
+
+function magicSplit($regex, $string) {
+    $pieces = preg_split(regex('(?:(?&string)|(?&comment)|(?&code))(*SKIP)(*FAIL)|' . $regex), $string);
+
+    foreach ($pieces as &$piece) {
+        $piece = trim($piece);
+    }
+
+    if ($pieces === ['']) {
+        return [];
+    }
+
+    return $pieces;
+}

grammar/rebuildParsers.php

@@ -1,66 +1,45 @@
-<?php
+<?php declare(strict_types=1);
 
-$grammarFileToName = [
+require __DIR__ . '/phpyLang.php';
-    __DIR__ . '/php5.y' => 'Php5',
+
-    __DIR__ . '/php7.y' => 'Php7',
+$parserToDefines = [
+    'Php7' => ['PHP7' => true],
+    'Php8' => ['PHP8' => true],
 ];
 
-$tokensFile     = __DIR__ . '/tokens.y';
+$grammarFile    = __DIR__ . '/php.y';
-$tokensTemplate = __DIR__ . '/tokens.template';
 $skeletonFile   = __DIR__ . '/parser.template';
 $tmpGrammarFile = __DIR__ . '/tmp_parser.phpy';
 $tmpResultFile  = __DIR__ . '/tmp_parser.php';
 $resultDir = __DIR__ . '/../lib/PhpParser/Parser';
-$tokensResultsFile = $resultDir . '/Tokens.php';
 
-// check for kmyacc.exe binary in this directory, otherwise fall back to global name
+$kmyacc = getenv('KMYACC');
-$kmyacc = __DIR__ . '/kmyacc.exe';
+if (!$kmyacc) {
-if (!file_exists($kmyacc)) {
+    // Use phpyacc from dev dependencies by default.
-    $kmyacc = 'kmyacc';
+    $kmyacc = __DIR__ . '/../vendor/bin/phpyacc';
 }
 
 $options = array_flip($argv);
 $optionDebug = isset($options['--debug']);
 $optionKeepTmpGrammar = isset($options['--keep-tmp-grammar']);
 
-///////////////////////////////
-/// Utility regex constants ///
-///////////////////////////////
-
-const LIB = '(?(DEFINE)
-    (?<singleQuotedString>\'[^\\\\\']*+(?:\\\\.[^\\\\\']*+)*+\')
-    (?<doubleQuotedString>"[^\\\\"]*+(?:\\\\.[^\\\\"]*+)*+")
-    (?<string>(?&singleQuotedString)|(?&doubleQuotedString))
-    (?<comment>/\*[^*]*+(?:\*(?!/)[^*]*+)*+\*/)
-    (?<code>\{[^\'"/{}]*+(?:(?:(?&string)|(?&comment)|(?&code)|/)[^\'"/{}]*+)*+})
-)';
-
-const PARAMS = '\[(?<params>[^[\]]*+(?:\[(?&params)\][^[\]]*+)*+)\]';
-const ARGS   = '\((?<args>[^()]*+(?:\((?&args)\)[^()]*+)*+)\)';
-
 ///////////////////
 /// Main script ///
 ///////////////////
 
-$tokens = file_get_contents($tokensFile);
+foreach ($parserToDefines as $name => $defines) {
-
-foreach ($grammarFileToName as $grammarFile => $name) {
     echo "Building temporary $name grammar file.\n";
 
     $grammarCode = file_get_contents($grammarFile);
-    $grammarCode = str_replace('%tokens', $tokens, $grammarCode);
+    $grammarCode = replaceIfBlocks($grammarCode, $defines);
-
+    $grammarCode = preprocessGrammar($grammarCode);
-    $grammarCode = resolveNodes($grammarCode);
-    $grammarCode = resolveMacros($grammarCode);
-    $grammarCode = resolveStackAccess($grammarCode);
 
     file_put_contents($tmpGrammarFile, $grammarCode);
 
     $additionalArgs = $optionDebug ? '-t -v' : '';
 
     echo "Building $name parser.\n";
-    $output = trim(shell_exec("$kmyacc $additionalArgs -l -m $skeletonFile -p $name $tmpGrammarFile 2>&1"));
+    $output = execCmd("$kmyacc $additionalArgs -m $skeletonFile -p $name $tmpGrammarFile");
-    echo "Output: \"$output\"\n";
 
     $resultCode = file_get_contents($tmpResultFile);
     $resultCode = removeTrailingWhitespace($resultCode);
@@ -69,152 +48,14 @@ foreach ($grammarFileToName as $grammarFile => $name) {
     file_put_contents("$resultDir/$name.php", $resultCode);
     unlink($tmpResultFile);
 
-    echo "Building token definition.\n";
-    $output = trim(shell_exec("$kmyacc -l -m $tokensTemplate $tmpGrammarFile 2>&1"));
-    assert($output === '');
-    rename($tmpResultFile, $tokensResultsFile);
-
     if (!$optionKeepTmpGrammar) {
         unlink($tmpGrammarFile);
     }
 }
 
-///////////////////////////////
+////////////////////////////////
-/// Preprocessing functions ///
+/// Utility helper functions ///
-///////////////////////////////
+////////////////////////////////
-
-function resolveNodes($code) {
-    return preg_replace_callback(
-        '~\b(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
-        function($matches) {
-            // recurse
-            $matches['params'] = resolveNodes($matches['params']);
-
-            $params = magicSplit(
-                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
-                $matches['params']
-            );
-
-            $paramCode = '';
-            foreach ($params as $param) {
-                $paramCode .= $param . ', ';
-            }
-
-            return 'new ' . $matches['name'] . '(' . $paramCode . 'attributes())';
-        },
-        $code
-    );
-}
-
-function resolveMacros($code) {
-    return preg_replace_callback(
-        '~\b(?<!::|->)(?!array\()(?<name>[a-z][A-Za-z]++)' . ARGS . '~',
-        function($matches) {
-            // recurse
-            $matches['args'] = resolveMacros($matches['args']);
-
-            $name = $matches['name'];
-            $args = magicSplit(
-                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
-                $matches['args']
-            );
-
-            if ('attributes' == $name) {
-                assertArgs(0, $args, $name);
-                return '$this->startAttributeStack[#1] + $this->endAttributes';
-            }
-
-            if ('stackAttributes' == $name) {
-                assertArgs(1, $args, $name);
-                return '$this->startAttributeStack[' . $args[0] . ']'
-                     . ' + $this->endAttributeStack[' . $args[0] . ']';
-            }
-
-            if ('init' == $name) {
-                return '$$ = array(' . implode(', ', $args) . ')';
-            }
-
-            if ('push' == $name) {
-                assertArgs(2, $args, $name);
-
-                return $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0];
-            }
-
-            if ('pushNormalizing' == $name) {
-                assertArgs(2, $args, $name);
-
-                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
-                     . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
-            }
-
-            if ('toArray' == $name) {
-                assertArgs(1, $args, $name);
-
-                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
-            }
-
-            if ('parseVar' == $name) {
-                assertArgs(1, $args, $name);
-
-                return 'substr(' . $args[0] . ', 1)';
-            }
-
-            if ('parseEncapsed' == $name) {
-                assertArgs(3, $args, $name);
-
-                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\Scalar\EncapsedStringPart) {'
-                     . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, ' . $args[1] . ', ' . $args[2] . '); } }';
-            }
-
-            if ('makeNop' == $name) {
-                assertArgs(3, $args, $name);
-
-                return '$startAttributes = ' . $args[1] . ';'
-                . ' if (isset($startAttributes[\'comments\']))'
-                . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
-                . ' else { ' . $args[0] . ' = null; }';
-            }
-
-            if ('strKind' == $name) {
-                assertArgs(1, $args, $name);
-
-                return '(' . $args[0] . '[0] === "\'" || (' . $args[0] . '[1] === "\'" && '
-                     . '(' . $args[0] . '[0] === \'b\' || ' . $args[0] . '[0] === \'B\')) '
-                     . '? Scalar\String_::KIND_SINGLE_QUOTED : Scalar\String_::KIND_DOUBLE_QUOTED)';
-            }
-
-            if ('prependLeadingComments' == $name) {
-                assertArgs(1, $args, $name);
-
-                return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
-                . 'if (!empty($attrs[\'comments\'])) {'
-                . '$stmts[0]->setAttribute(\'comments\', '
-                . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
-            }
-
-            return $matches[0];
-        },
-        $code
-    );
-}
-
-function assertArgs($num, $args, $name) {
-    if ($num != count($args)) {
-        die('Wrong argument count for ' . $name . '().');
-    }
-}
-
-function resolveStackAccess($code) {
-    $code = preg_replace('/\$\d+/', '$this->semStack[$0]', $code);
-    $code = preg_replace('/#(\d+)/', '$$1', $code);
-    return $code;
-}
-
-function removeTrailingWhitespace($code) {
-    $lines = explode("\n", $code);
-    $lines = array_map('rtrim', $lines);
-    return implode("\n", $lines);
-}
 
 function ensureDirExists($dir) {
     if (!is_dir($dir)) {
@@ -222,24 +63,18 @@ function ensureDirExists($dir) {
     }
 }
 
-//////////////////////////////
+function execCmd($cmd) {
-/// Regex helper functions ///
+    $output = trim(shell_exec("$cmd 2>&1") ?? '');
-//////////////////////////////
+    if ($output !== "") {
-
+        echo "> " . $cmd . "\n";
-function regex($regex) {
+        echo $output;
-    return '~' . LIB . '(?:' . str_replace('~', '\~', $regex) . ')~';
+    }
+    return $output;
 }
 
-function magicSplit($regex, $string) {
+function replaceIfBlocks(string $code, array $defines): string {
-    $pieces = preg_split(regex('(?:(?&string)|(?&comment)|(?&code))(*SKIP)(*FAIL)|' . $regex), $string);
+    return preg_replace_callback('/\n#if\s+(\w+)\n(.*?)\n#endif/s', function ($matches) use ($defines) {
-
+        $value = $defines[$matches[1]] ?? false;
-    foreach ($pieces as &$piece) {
+        return $value ? $matches[2] : '';
-        $piece = trim($piece);
+    }, $code);
-    }
-
-    if ($pieces === ['']) {
-        return [];
-    }
-
-    return $pieces;
 }

grammar/tokens.template

@@ -1,17 +0,0 @@
-<?php
-$meta #
-#semval($) $this->semValue
-#semval($,%t) $this->semValue
-#semval(%n) $this->stackPos-(%l-%n)
-#semval(%n,%t) $this->stackPos-(%l-%n)
-
-namespace PhpParser\Parser;
-#include;
-
-/* GENERATED file based on grammar/tokens.y */
-final class Tokens
-{
-#tokenval
-    const %s = %n;
-#endtokenval
-}

grammar/tokens.y

@@ -1,113 +0,0 @@
-/* We currently rely on the token ID mapping to be the same between PHP 5 and PHP 7 - so the same lexer can be used for
- * both. This is enforced by sharing this token file. */
-
-%left T_INCLUDE T_INCLUDE_ONCE T_EVAL T_REQUIRE T_REQUIRE_ONCE
-%left ','
-%left T_LOGICAL_OR
-%left T_LOGICAL_XOR
-%left T_LOGICAL_AND
-%right T_PRINT
-%right T_YIELD
-%right T_DOUBLE_ARROW
-%right T_YIELD_FROM
-%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL
-%left '?' ':'
-%right T_COALESCE
-%left T_BOOLEAN_OR
-%left T_BOOLEAN_AND
-%left '|'
-%left '^'
-%left '&'
-%nonassoc T_IS_EQUAL T_IS_NOT_EQUAL T_IS_IDENTICAL T_IS_NOT_IDENTICAL T_SPACESHIP
-%nonassoc '<' T_IS_SMALLER_OR_EQUAL '>' T_IS_GREATER_OR_EQUAL
-%left T_SL T_SR
-%left '+' '-' '.'
-%left '*' '/' '%'
-%right '!'
-%nonassoc T_INSTANCEOF
-%right '~' T_INC T_DEC T_INT_CAST T_DOUBLE_CAST T_STRING_CAST T_ARRAY_CAST T_OBJECT_CAST T_BOOL_CAST T_UNSET_CAST '@'
-%right T_POW
-%right '['
-%nonassoc T_NEW T_CLONE
-%token T_EXIT
-%token T_IF
-%left T_ELSEIF
-%left T_ELSE
-%left T_ENDIF
-%token T_LNUMBER
-%token T_DNUMBER
-%token T_STRING
-%token T_STRING_VARNAME
-%token T_VARIABLE
-%token T_NUM_STRING
-%token T_INLINE_HTML
-%token T_CHARACTER
-%token T_BAD_CHARACTER
-%token T_ENCAPSED_AND_WHITESPACE
-%token T_CONSTANT_ENCAPSED_STRING
-%token T_ECHO
-%token T_DO
-%token T_WHILE
-%token T_ENDWHILE
-%token T_FOR
-%token T_ENDFOR
-%token T_FOREACH
-%token T_ENDFOREACH
-%token T_DECLARE
-%token T_ENDDECLARE
-%token T_AS
-%token T_SWITCH
-%token T_ENDSWITCH
-%token T_CASE
-%token T_DEFAULT
-%token T_BREAK
-%token T_CONTINUE
-%token T_GOTO
-%token T_FUNCTION
-%token T_CONST
-%token T_RETURN
-%token T_TRY
-%token T_CATCH
-%token T_FINALLY
-%token T_THROW
-%token T_USE
-%token T_INSTEADOF
-%token T_GLOBAL
-%right T_STATIC T_ABSTRACT T_FINAL T_PRIVATE T_PROTECTED T_PUBLIC
-%token T_VAR
-%token T_UNSET
-%token T_ISSET
-%token T_EMPTY
-%token T_HALT_COMPILER
-%token T_CLASS
-%token T_TRAIT
-%token T_INTERFACE
-%token T_EXTENDS
-%token T_IMPLEMENTS
-%token T_OBJECT_OPERATOR
-%token T_DOUBLE_ARROW
-%token T_LIST
-%token T_ARRAY
-%token T_CALLABLE
-%token T_CLASS_C
-%token T_TRAIT_C
-%token T_METHOD_C
-%token T_FUNC_C
-%token T_LINE
-%token T_FILE
-%token T_COMMENT
-%token T_DOC_COMMENT
-%token T_OPEN_TAG
-%token T_OPEN_TAG_WITH_ECHO
-%token T_CLOSE_TAG
-%token T_WHITESPACE
-%token T_START_HEREDOC
-%token T_END_HEREDOC
-%token T_DOLLAR_OPEN_CURLY_BRACES
-%token T_CURLY_OPEN
-%token T_PAAMAYIM_NEKUDOTAYIM
-%token T_NAMESPACE
-%token T_NS_C
-%token T_DIR
-%token T_NS_SEPARATOR
-%token T_ELLIPSIS

lib/PhpParser/Builder.php

@@ -2,12 +2,11 @@
 
 namespace PhpParser;
 
-interface Builder
+interface Builder {
-{
     /**
      * Returns the built node.
      *
      * @return Node The built node
      */
-    public function getNode() : Node;
+    public function getNode(): Node;
 }

lib/PhpParser/Builder/ClassConst.php

@@ -0,0 +1,132 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PhpParser\Builder;
+
+use PhpParser;
+use PhpParser\BuilderHelpers;
+use PhpParser\Modifiers;
+use PhpParser\Node;
+use PhpParser\Node\Const_;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Stmt;
+
+class ClassConst implements PhpParser\Builder {
+    protected $flags = 0;
+    protected $attributes = [];
+    protected $constants = [];
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
+    /**
+     * Creates a class constant builder
+     *
+     * @param string|Identifier                          $name  Name
+     * @param Node\Expr|bool|null|int|float|string|array $value Value
+     */
+    public function __construct($name, $value) {
+        $this->constants = [new Const_($name, BuilderHelpers::normalizeValue($value))];
+    }
+
+    /**
+     * Add another constant to const group
+     *
+     * @param string|Identifier                          $name  Name
+     * @param Node\Expr|bool|null|int|float|string|array $value Value
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addConst($name, $value) {
+        $this->constants[] = new Const_($name, BuilderHelpers::normalizeValue($value));
+
+        return $this;
+    }
+
+    /**
+     * Makes the constant public.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makePublic() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PUBLIC);
+
+        return $this;
+    }
+
+    /**
+     * Makes the constant protected.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeProtected() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED);
+
+        return $this;
+    }
+
+    /**
+     * Makes the constant private.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makePrivate() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE);
+
+        return $this;
+    }
+
+    /**
+     * Makes the constant final.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeFinal() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::FINAL);
+
+        return $this;
+    }
+
+    /**
+     * Sets doc comment for the constant.
+     *
+     * @param PhpParser\Comment\Doc|string $docComment Doc comment to set
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function setDocComment($docComment) {
+        $this->attributes = [
+            'comments' => [BuilderHelpers::normalizeDocComment($docComment)]
+        ];
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
+    /**
+     * Returns the built class node.
+     *
+     * @return Stmt\ClassConst The built constant node
+     */
+    public function getNode(): PhpParser\Node {
+        return new Stmt\ClassConst(
+            $this->constants,
+            $this->flags,
+            $this->attributes,
+            $this->attributeGroups
+        );
+    }
+}

lib/PhpParser/Builder/Class_.php

@@ -4,11 +4,12 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Modifiers;
+use PhpParser\Node;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
-class Class_ extends Declaration
+class Class_ extends Declaration {
-{
     protected $name;
 
     protected $extends = null;
@@ -20,6 +21,9 @@ class Class_ extends Declaration
     protected $properties = [];
     protected $methods = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a class builder.
      *
@@ -63,7 +67,7 @@ class Class_ extends Declaration
      * @return $this The builder instance (for fluid interface)
      */
     public function makeAbstract() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_ABSTRACT);
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Modifiers::ABSTRACT);
 
         return $this;
     }
@@ -74,7 +78,13 @@ class Class_ extends Declaration
      * @return $this The builder instance (for fluid interface)
      */
     public function makeFinal() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Modifiers::FINAL);
+
+        return $this;
+    }
+
+    public function makeReadonly() {
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Modifiers::READONLY);
 
         return $this;
     }
@@ -106,17 +116,31 @@ class Class_ extends Declaration
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built class node.
      *
      * @return Stmt\Class_ The built class node
      */
-    public function getNode() : PhpParser\Node {
+    public function getNode(): PhpParser\Node {
         return new Stmt\Class_($this->name, [
             'flags' => $this->flags,
             'extends' => $this->extends,
             'implements' => $this->implements,
             'stmts' => array_merge($this->uses, $this->constants, $this->properties, $this->methods),
+            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Declaration.php

@@ -5,8 +5,7 @@ namespace PhpParser\Builder;
 use PhpParser;
 use PhpParser\BuilderHelpers;
 
-abstract class Declaration implements PhpParser\Builder
+abstract class Declaration implements PhpParser\Builder {
-{
     protected $attributes = [];
 
     abstract public function addStmt($stmt);

lib/PhpParser/Builder/EnumCase.php

@@ -0,0 +1,84 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PhpParser\Builder;
+
+use PhpParser;
+use PhpParser\BuilderHelpers;
+use PhpParser\Node;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Stmt;
+
+class EnumCase implements PhpParser\Builder {
+    protected $name;
+    protected $value = null;
+    protected $attributes = [];
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
+    /**
+     * Creates an enum case builder.
+     *
+     * @param string|Identifier $name  Name
+     */
+    public function __construct($name) {
+        $this->name = $name;
+    }
+
+    /**
+     * Sets the value.
+     *
+     * @param Node\Expr|string|int $value
+     *
+     * @return $this
+     */
+    public function setValue($value) {
+        $this->value = BuilderHelpers::normalizeValue($value);
+
+        return $this;
+    }
+
+    /**
+     * Sets doc comment for the constant.
+     *
+     * @param PhpParser\Comment\Doc|string $docComment Doc comment to set
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function setDocComment($docComment) {
+        $this->attributes = [
+            'comments' => [BuilderHelpers::normalizeDocComment($docComment)]
+        ];
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
+    /**
+     * Returns the built enum case node.
+     *
+     * @return Stmt\EnumCase The built constant node
+     */
+    public function getNode(): PhpParser\Node {
+        return new Stmt\EnumCase(
+            $this->name,
+            $this->value,
+            $this->attributes,
+            $this->attributeGroups
+        );
+    }
+}

lib/PhpParser/Builder/Enum_.php

@@ -0,0 +1,116 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Builder;
+
+use PhpParser;
+use PhpParser\BuilderHelpers;
+use PhpParser\Node;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Name;
+use PhpParser\Node\Stmt;
+
+class Enum_ extends Declaration {
+    protected $name;
+    protected $scalarType = null;
+
+    protected $implements = [];
+
+    protected $uses = [];
+    protected $enumCases = [];
+    protected $constants = [];
+    protected $methods = [];
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
+    /**
+     * Creates an enum builder.
+     *
+     * @param string $name Name of the enum
+     */
+    public function __construct(string $name) {
+        $this->name = $name;
+    }
+
+    /**
+     * Sets the scalar type.
+     *
+     * @param string|Identifier $type
+     *
+     * @return $this
+     */
+    public function setScalarType($scalarType) {
+        $this->scalarType = BuilderHelpers::normalizeType($scalarType);
+
+        return $this;
+    }
+
+    /**
+     * Implements one or more interfaces.
+     *
+     * @param Name|string ...$interfaces Names of interfaces to implement
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function implement(...$interfaces) {
+        foreach ($interfaces as $interface) {
+            $this->implements[] = BuilderHelpers::normalizeName($interface);
+        }
+
+        return $this;
+    }
+
+    /**
+     * Adds a statement.
+     *
+     * @param Stmt|PhpParser\Builder $stmt The statement to add
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addStmt($stmt) {
+        $stmt = BuilderHelpers::normalizeNode($stmt);
+
+        $targets = [
+            Stmt\TraitUse::class    => &$this->uses,
+            Stmt\EnumCase::class    => &$this->enumCases,
+            Stmt\ClassConst::class  => &$this->constants,
+            Stmt\ClassMethod::class => &$this->methods,
+        ];
+
+        $class = \get_class($stmt);
+        if (!isset($targets[$class])) {
+            throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
+        }
+
+        $targets[$class][] = $stmt;
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
+    /**
+     * Returns the built class node.
+     *
+     * @return Stmt\Enum_ The built enum node
+     */
+    public function getNode(): PhpParser\Node {
+        return new Stmt\Enum_($this->name, [
+            'scalarType' => $this->scalarType,
+            'implements' => $this->implements,
+            'stmts' => array_merge($this->uses, $this->enumCases, $this->constants, $this->methods),
+            'attrGroups' => $this->attributeGroups,
+        ], $this->attributes);
+    }
+}

lib/PhpParser/Builder/FunctionLike.php

@@ -5,8 +5,7 @@ namespace PhpParser\Builder;
 use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 
-abstract class FunctionLike extends Declaration
+abstract class FunctionLike extends Declaration {
-{
     protected $returnByRef = false;
     protected $params = [];
 
@@ -61,8 +60,7 @@ abstract class FunctionLike extends Declaration
     /**
      * Sets the return type for PHP 7.
      *
-     * @param string|Node\Name|Node\NullableType $type One of array, callable, string, int, float,
+     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type
-     *                                                 bool, iterable, or a class/interface name.
      *
      * @return $this The builder instance (for fluid interface)
      */

lib/PhpParser/Builder/Function_.php

@@ -7,11 +7,13 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Function_ extends FunctionLike
+class Function_ extends FunctionLike {
-{
     protected $name;
     protected $stmts = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a function builder.
      *
@@ -34,17 +36,31 @@ class Function_ extends FunctionLike
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built function node.
      *
      * @return Stmt\Function_ The built function node
      */
-    public function getNode() : Node {
+    public function getNode(): Node {
         return new Stmt\Function_($this->name, [
             'byRef'      => $this->returnByRef,
             'params'     => $this->params,
             'returnType' => $this->returnType,
             'stmts'      => $this->stmts,
+            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Interface_.php

@@ -4,16 +4,19 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Node;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
-class Interface_ extends Declaration
+class Interface_ extends Declaration {
-{
     protected $name;
     protected $extends = [];
     protected $constants = [];
     protected $methods = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates an interface builder.
      *
@@ -61,15 +64,29 @@ class Interface_ extends Declaration
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built interface node.
      *
      * @return Stmt\Interface_ The built interface node
      */
-    public function getNode() : PhpParser\Node {
+    public function getNode(): PhpParser\Node {
         return new Stmt\Interface_($this->name, [
             'extends' => $this->extends,
             'stmts' => array_merge($this->constants, $this->methods),
+            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Method.php

@@ -4,17 +4,20 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Method extends FunctionLike
+class Method extends FunctionLike {
-{
     protected $name;
     protected $flags = 0;
 
     /** @var array|null */
     protected $stmts = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a method builder.
      *
@@ -30,7 +33,7 @@ class Method extends FunctionLike
      * @return $this The builder instance (for fluid interface)
      */
     public function makePublic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PUBLIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PUBLIC);
 
         return $this;
     }
@@ -41,7 +44,7 @@ class Method extends FunctionLike
      * @return $this The builder instance (for fluid interface)
      */
     public function makeProtected() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PROTECTED);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED);
 
         return $this;
     }
@@ -52,7 +55,7 @@ class Method extends FunctionLike
      * @return $this The builder instance (for fluid interface)
      */
     public function makePrivate() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PRIVATE);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE);
 
         return $this;
     }
@@ -63,7 +66,7 @@ class Method extends FunctionLike
      * @return $this The builder instance (for fluid interface)
      */
     public function makeStatic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_STATIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::STATIC);
 
         return $this;
     }
@@ -78,7 +81,7 @@ class Method extends FunctionLike
             throw new \LogicException('Cannot make method with statements abstract');
         }
 
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_ABSTRACT);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::ABSTRACT);
         $this->stmts = null; // abstract methods don't have statements
 
         return $this;
@@ -90,7 +93,7 @@ class Method extends FunctionLike
      * @return $this The builder instance (for fluid interface)
      */
     public function makeFinal() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::FINAL);
 
         return $this;
     }
@@ -112,18 +115,32 @@ class Method extends FunctionLike
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built method node.
      *
      * @return Stmt\ClassMethod The built method node
      */
-    public function getNode() : Node {
+    public function getNode(): Node {
         return new Stmt\ClassMethod($this->name, [
             'flags'      => $this->flags,
             'byRef'      => $this->returnByRef,
             'params'     => $this->params,
             'returnType' => $this->returnType,
             'stmts'      => $this->stmts,
+            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Namespace_.php

@@ -7,8 +7,7 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Namespace_ extends Declaration
+class Namespace_ extends Declaration {
-{
     private $name;
     private $stmts = [];
 
@@ -37,9 +36,9 @@ class Namespace_ extends Declaration
     /**
      * Returns the built node.
      *
-     * @return Node The built node
+     * @return Stmt\Namespace_ The built node
      */
-    public function getNode() : Node {
+    public function getNode(): Node {
         return new Stmt\Namespace_($this->name, $this->stmts, $this->attributes);
     }
 }

lib/PhpParser/Builder/Param.php

@@ -6,19 +6,21 @@ use PhpParser;
 use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 
-class Param implements PhpParser\Builder
+class Param implements PhpParser\Builder {
-{
     protected $name;
 
     protected $default = null;
 
-    /** @var string|Node\Name|Node\NullableType|null */
+    /** @var Node\Identifier|Node\Name|Node\NullableType|null */
     protected $type = null;
 
     protected $byRef = false;
 
     protected $variadic = false;
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a parameter builder.
      *
@@ -44,7 +46,7 @@ class Param implements PhpParser\Builder
     /**
      * Sets type for the parameter.
      *
-     * @param string|Node\Name|Node\NullableType $type Parameter type
+     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type Parameter type
      *
      * @return $this The builder instance (for fluid interface)
      */
@@ -57,19 +59,6 @@ class Param implements PhpParser\Builder
         return $this;
     }
 
-    /**
-     * Sets type for the parameter.
-     *
-     * @param string|Node\Name|Node\NullableType $type Parameter type
-     *
-     * @return $this The builder instance (for fluid interface)
-     *
-     * @deprecated Use setType() instead
-     */
-    public function setTypeHint($type) {
-        return $this->setType($type);
-    }
-
     /**
      * Make the parameter accept the value by reference.
      *
@@ -92,15 +81,28 @@ class Param implements PhpParser\Builder
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built parameter node.
      *
      * @return Node\Param The built parameter node
      */
-    public function getNode() : Node {
+    public function getNode(): Node {
         return new Node\Param(
             new Node\Expr\Variable($this->name),
-            $this->default, $this->type, $this->byRef, $this->variadic
+            $this->default, $this->type, $this->byRef, $this->variadic, [], 0, $this->attributeGroups
         );
     }
 }

lib/PhpParser/Builder/Property.php

@@ -4,13 +4,14 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Modifiers;
+use PhpParser\Node;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
-use PhpParser\Node\NullableType;
 use PhpParser\Node\Stmt;
+use PhpParser\Node\ComplexType;
 
-class Property implements PhpParser\Builder
+class Property implements PhpParser\Builder {
-{
     protected $name;
 
     protected $flags = 0;
@@ -20,6 +21,9 @@ class Property implements PhpParser\Builder
     /** @var null|Identifier|Name|NullableType */
     protected $type;
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a property builder.
      *
@@ -35,7 +39,7 @@ class Property implements PhpParser\Builder
      * @return $this The builder instance (for fluid interface)
      */
     public function makePublic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PUBLIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PUBLIC);
 
         return $this;
     }
@@ -46,7 +50,7 @@ class Property implements PhpParser\Builder
      * @return $this The builder instance (for fluid interface)
      */
     public function makeProtected() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PROTECTED);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED);
 
         return $this;
     }
@@ -57,7 +61,7 @@ class Property implements PhpParser\Builder
      * @return $this The builder instance (for fluid interface)
      */
     public function makePrivate() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PRIVATE);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE);
 
         return $this;
     }
@@ -68,7 +72,18 @@ class Property implements PhpParser\Builder
      * @return $this The builder instance (for fluid interface)
      */
     public function makeStatic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_STATIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::STATIC);
+
+        return $this;
+    }
+
+    /**
+     * Makes the property readonly.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeReadonly() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::READONLY);
 
         return $this;
     }
@@ -104,7 +119,7 @@ class Property implements PhpParser\Builder
     /**
      * Sets the property type for PHP 7.4+.
      *
-     * @param string|Name|NullableType|Identifier $type
+     * @param string|Name|Identifier|ComplexType $type
      *
      * @return $this
      */
@@ -114,19 +129,33 @@ class Property implements PhpParser\Builder
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built class node.
      *
      * @return Stmt\Property The built property node
      */
-    public function getNode() : PhpParser\Node {
+    public function getNode(): PhpParser\Node {
         return new Stmt\Property(
-            $this->flags !== 0 ? $this->flags : Stmt\Class_::MODIFIER_PUBLIC,
+            $this->flags !== 0 ? $this->flags : Modifiers::PUBLIC,
             [
-                new Stmt\PropertyProperty($this->name, $this->default)
+                new Node\PropertyItem($this->name, $this->default)
             ],
             $this->attributes,
-            $this->type
+            $this->type,
+            $this->attributeGroups
         );
     }
 }

lib/PhpParser/Builder/TraitUse.php

@@ -7,8 +7,7 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class TraitUse implements Builder
+class TraitUse implements Builder {
-{
     protected $traits = [];
     protected $adaptations = [];
 
@@ -58,7 +57,7 @@ class TraitUse implements Builder
      *
      * @return Node The built node
      */
-    public function getNode() : Node {
+    public function getNode(): Node {
         return new Stmt\TraitUse($this->traits, $this->adaptations);
     }
 }

lib/PhpParser/Builder/TraitUseAdaptation.php

@@ -4,14 +4,14 @@ namespace PhpParser\Builder;
 
 use PhpParser\Builder;
 use PhpParser\BuilderHelpers;
+use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class TraitUseAdaptation implements Builder
+class TraitUseAdaptation implements Builder {
-{
+    public const TYPE_UNDEFINED  = 0;
-    const TYPE_UNDEFINED  = 0;
+    public const TYPE_ALIAS      = 1;
-    const TYPE_ALIAS      = 1;
+    public const TYPE_PRECEDENCE = 2;
-    const TYPE_PRECEDENCE = 2;
 
     /** @var int Type of building adaptation */
     protected $type;
@@ -33,7 +33,7 @@ class TraitUseAdaptation implements Builder
     public function __construct($trait, $method) {
         $this->type = self::TYPE_UNDEFINED;
 
-        $this->trait = is_null($trait)? null: BuilderHelpers::normalizeName($trait);
+        $this->trait = is_null($trait) ? null : BuilderHelpers::normalizeName($trait);
         $this->method = BuilderHelpers::normalizeIdentifier($method);
     }
 
@@ -63,7 +63,7 @@ class TraitUseAdaptation implements Builder
      * @return $this The builder instance (for fluid interface)
      */
     public function makePublic() {
-        $this->setModifier(Stmt\Class_::MODIFIER_PUBLIC);
+        $this->setModifier(Modifiers::PUBLIC);
         return $this;
     }
 
@@ -73,7 +73,7 @@ class TraitUseAdaptation implements Builder
      * @return $this The builder instance (for fluid interface)
      */
     public function makeProtected() {
-        $this->setModifier(Stmt\Class_::MODIFIER_PROTECTED);
+        $this->setModifier(Modifiers::PROTECTED);
         return $this;
     }
 
@@ -83,7 +83,7 @@ class TraitUseAdaptation implements Builder
      * @return $this The builder instance (for fluid interface)
      */
     public function makePrivate() {
-        $this->setModifier(Stmt\Class_::MODIFIER_PRIVATE);
+        $this->setModifier(Modifiers::PRIVATE);
         return $this;
     }
 
@@ -135,7 +135,7 @@ class TraitUseAdaptation implements Builder
      *
      * @return Node The built node
      */
-    public function getNode() : Node {
+    public function getNode(): Node {
         switch ($this->type) {
             case self::TYPE_ALIAS:
                 return new Stmt\TraitUseAdaptation\Alias($this->trait, $this->method, $this->modifier, $this->alias);

lib/PhpParser/Builder/Trait_.php

@@ -4,15 +4,18 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Trait_ extends Declaration
+class Trait_ extends Declaration {
-{
     protected $name;
     protected $uses = [];
     protected $properties = [];
     protected $methods = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates an interface builder.
      *
@@ -45,15 +48,29 @@ class Trait_ extends Declaration
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built trait node.
      *
      * @return Stmt\Trait_ The built interface node
      */
-    public function getNode() : PhpParser\Node {
+    public function getNode(): PhpParser\Node {
         return new Stmt\Trait_(
             $this->name, [
-                'stmts' => array_merge($this->uses, $this->properties, $this->methods)
+                'stmts' => array_merge($this->uses, $this->properties, $this->methods),
+                'attrGroups' => $this->attributeGroups,
             ], $this->attributes
         );
     }

lib/PhpParser/Builder/Use_.php

@@ -7,8 +7,7 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Use_ implements Builder
+class Use_ implements Builder {
-{
     protected $name;
     protected $type;
     protected $alias = null;
@@ -39,11 +38,11 @@ class Use_ implements Builder
     /**
      * Returns the built node.
      *
-     * @return Node The built node
+     * @return Stmt\Use_ The built node
      */
-    public function getNode() : Node {
+    public function getNode(): Node {
         return new Stmt\Use_([
-            new Stmt\UseUse($this->name, $this->alias)
+            new Node\UseItem($this->name, $this->alias)
         ], $this->type);
     }
 }

lib/PhpParser/BuilderFactory.php

@@ -8,11 +8,24 @@ use PhpParser\Node\Expr\BinaryOp\Concat;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
 use PhpParser\Node\Scalar\String_;
-use PhpParser\Node\Stmt;
 use PhpParser\Node\Stmt\Use_;
 
-class BuilderFactory
+class BuilderFactory {
-{
+    /**
+     * Creates an attribute node.
+     *
+     * @param string|Name $name Name of the attribute
+     * @param array       $args Attribute named arguments
+     *
+     * @return Node\Attribute
+     */
+    public function attribute($name, array $args = []): Node\Attribute {
+        return new Node\Attribute(
+            BuilderHelpers::normalizeName($name),
+            $this->args($args)
+        );
+    }
+
     /**
      * Creates a namespace builder.
      *
@@ -20,7 +33,7 @@ class BuilderFactory
      *
      * @return Builder\Namespace_ The created namespace builder
      */
-    public function namespace($name) : Builder\Namespace_ {
+    public function namespace($name): Builder\Namespace_ {
         return new Builder\Namespace_($name);
     }
 
@@ -31,7 +44,7 @@ class BuilderFactory
      *
      * @return Builder\Class_ The created class builder
      */
-    public function class(string $name) : Builder\Class_ {
+    public function class(string $name): Builder\Class_ {
         return new Builder\Class_($name);
     }
 
@@ -42,7 +55,7 @@ class BuilderFactory
      *
      * @return Builder\Interface_ The created interface builder
      */
-    public function interface(string $name) : Builder\Interface_ {
+    public function interface(string $name): Builder\Interface_ {
         return new Builder\Interface_($name);
     }
 
@@ -53,10 +66,21 @@ class BuilderFactory
      *
      * @return Builder\Trait_ The created trait builder
      */
-    public function trait(string $name) : Builder\Trait_ {
+    public function trait(string $name): Builder\Trait_ {
         return new Builder\Trait_($name);
     }
 
+    /**
+     * Creates an enum builder.
+     *
+     * @param string $name Name of the enum
+     *
+     * @return Builder\Enum_ The created enum builder
+     */
+    public function enum(string $name): Builder\Enum_ {
+        return new Builder\Enum_($name);
+    }
+
     /**
      * Creates a trait use builder.
      *
@@ -64,7 +88,7 @@ class BuilderFactory
      *
      * @return Builder\TraitUse The create trait use builder
      */
-    public function useTrait(...$traits) : Builder\TraitUse {
+    public function useTrait(...$traits): Builder\TraitUse {
         return new Builder\TraitUse(...$traits);
     }
 
@@ -76,8 +100,8 @@ class BuilderFactory
      *
      * @return Builder\TraitUseAdaptation The create trait use adaptation builder
      */
-    public function traitUseAdaptation($trait, $method = null) : Builder\TraitUseAdaptation {
+    public function traitUseAdaptation($trait, $method = null): Builder\TraitUseAdaptation {
-        if (is_null($method)) {
+        if ($method === null) {
             $method = $trait;
             $trait = null;
         }
@@ -92,7 +116,7 @@ class BuilderFactory
      *
      * @return Builder\Method The created method builder
      */
-    public function method(string $name) : Builder\Method {
+    public function method(string $name): Builder\Method {
         return new Builder\Method($name);
     }
 
@@ -103,7 +127,7 @@ class BuilderFactory
      *
      * @return Builder\Param The created parameter builder
      */
-    public function param(string $name) : Builder\Param {
+    public function param(string $name): Builder\Param {
         return new Builder\Param($name);
     }
 
@@ -114,7 +138,7 @@ class BuilderFactory
      *
      * @return Builder\Property The created property builder
      */
-    public function property(string $name) : Builder\Property {
+    public function property(string $name): Builder\Property {
         return new Builder\Property($name);
     }
 
@@ -125,7 +149,7 @@ class BuilderFactory
      *
      * @return Builder\Function_ The created function builder
      */
-    public function function(string $name) : Builder\Function_ {
+    public function function(string $name): Builder\Function_ {
         return new Builder\Function_($name);
     }
 
@@ -136,7 +160,7 @@ class BuilderFactory
      *
      * @return Builder\Use_ The created use builder
      */
-    public function use($name) : Builder\Use_ {
+    public function use($name): Builder\Use_ {
         return new Builder\Use_($name, Use_::TYPE_NORMAL);
     }
 
@@ -147,7 +171,7 @@ class BuilderFactory
      *
      * @return Builder\Use_ The created use function builder
      */
-    public function useFunction($name) : Builder\Use_ {
+    public function useFunction($name): Builder\Use_ {
         return new Builder\Use_($name, Use_::TYPE_FUNCTION);
     }
 
@@ -158,10 +182,33 @@ class BuilderFactory
      *
      * @return Builder\Use_ The created use const builder
      */
-    public function useConst($name) : Builder\Use_ {
+    public function useConst($name): Builder\Use_ {
         return new Builder\Use_($name, Use_::TYPE_CONSTANT);
     }
 
+    /**
+     * Creates a class constant builder.
+     *
+     * @param string|Identifier                          $name  Name
+     * @param Node\Expr|bool|null|int|float|string|array $value Value
+     *
+     * @return Builder\ClassConst The created use const builder
+     */
+    public function classConst($name, $value): Builder\ClassConst {
+        return new Builder\ClassConst($name, $value);
+    }
+
+    /**
+     * Creates an enum case builder.
+     *
+     * @param string|Identifier $name  Name
+     *
+     * @return Builder\EnumCase The created use const builder
+     */
+    public function enumCase($name): Builder\EnumCase {
+        return new Builder\EnumCase($name);
+    }
+
     /**
      * Creates node a for a literal value.
      *
@@ -169,7 +216,7 @@ class BuilderFactory
      *
      * @return Expr
      */
-    public function val($value) : Expr {
+    public function val($value): Expr {
         return BuilderHelpers::normalizeValue($value);
     }
 
@@ -180,7 +227,7 @@ class BuilderFactory
      *
      * @return Expr\Variable
      */
-    public function var($name) : Expr\Variable {
+    public function var($name): Expr\Variable {
         if (!\is_string($name) && !$name instanceof Expr) {
             throw new \LogicException('Variable name must be string or Expr');
         }
@@ -197,14 +244,16 @@ class BuilderFactory
      *
      * @return Arg[]
      */
-    public function args(array $args) : array {
+    public function args(array $args): array {
         $normalizedArgs = [];
-        foreach ($args as $arg) {
+        foreach ($args as $key => $arg) {
-            if ($arg instanceof Arg) {
+            if (!($arg instanceof Arg)) {
-                $normalizedArgs[] = $arg;
+                $arg = new Arg(BuilderHelpers::normalizeValue($arg));
-            } else {
-                $normalizedArgs[] = new Arg(BuilderHelpers::normalizeValue($arg));
             }
+            if (\is_string($key)) {
+                $arg->name = BuilderHelpers::normalizeIdentifier($key);
+            }
+            $normalizedArgs[] = $arg;
         }
         return $normalizedArgs;
     }
@@ -217,7 +266,7 @@ class BuilderFactory
      *
      * @return Expr\FuncCall
      */
-    public function funcCall($name, array $args = []) : Expr\FuncCall {
+    public function funcCall($name, array $args = []): Expr\FuncCall {
         return new Expr\FuncCall(
             BuilderHelpers::normalizeNameOrExpr($name),
             $this->args($args)
@@ -233,7 +282,7 @@ class BuilderFactory
      *
      * @return Expr\MethodCall
      */
-    public function methodCall(Expr $var, $name, array $args = []) : Expr\MethodCall {
+    public function methodCall(Expr $var, $name, array $args = []): Expr\MethodCall {
         return new Expr\MethodCall(
             $var,
             BuilderHelpers::normalizeIdentifierOrExpr($name),
@@ -250,7 +299,7 @@ class BuilderFactory
      *
      * @return Expr\StaticCall
      */
-    public function staticCall($class, $name, array $args = []) : Expr\StaticCall {
+    public function staticCall($class, $name, array $args = []): Expr\StaticCall {
         return new Expr\StaticCall(
             BuilderHelpers::normalizeNameOrExpr($class),
             BuilderHelpers::normalizeIdentifierOrExpr($name),
@@ -266,7 +315,7 @@ class BuilderFactory
      *
      * @return Expr\New_
      */
-    public function new($class, array $args = []) : Expr\New_ {
+    public function new($class, array $args = []): Expr\New_ {
         return new Expr\New_(
             BuilderHelpers::normalizeNameOrExpr($class),
             $this->args($args)
@@ -280,7 +329,7 @@ class BuilderFactory
      *
      * @return Expr\ConstFetch
      */
-    public function constFetch($name) : Expr\ConstFetch {
+    public function constFetch($name): Expr\ConstFetch {
         return new Expr\ConstFetch(BuilderHelpers::normalizeName($name));
     }
 
@@ -292,7 +341,7 @@ class BuilderFactory
      *
      * @return Expr\PropertyFetch
      */
-    public function propertyFetch(Expr $var, $name) : Expr\PropertyFetch {
+    public function propertyFetch(Expr $var, $name): Expr\PropertyFetch {
         return new Expr\PropertyFetch($var, BuilderHelpers::normalizeIdentifierOrExpr($name));
     }
 
@@ -318,7 +367,7 @@ class BuilderFactory
      *
      * @return Concat
      */
-    public function concat(...$exprs) : Concat {
+    public function concat(...$exprs): Concat {
         $numExprs = count($exprs);
         if ($numExprs < 2) {
             throw new \LogicException('Expected at least two expressions');
@@ -335,7 +384,7 @@ class BuilderFactory
      * @param string|Expr $expr
      * @return Expr
      */
-    private function normalizeStringExpr($expr) : Expr {
+    private function normalizeStringExpr($expr): Expr {
         if ($expr instanceof Expr) {
             return $expr;
         }

lib/PhpParser/BuilderHelpers.php

@@ -2,6 +2,7 @@
 
 namespace PhpParser;
 
+use PhpParser\Node\ComplexType;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
@@ -14,8 +15,7 @@ use PhpParser\Node\Stmt;
  *
  * @internal
  */
-final class BuilderHelpers
+final class BuilderHelpers {
-{
     /**
      * Normalizes a node: Converts builder objects to nodes.
      *
@@ -23,10 +23,12 @@ final class BuilderHelpers
      *
      * @return Node The normalized node
      */
-    public static function normalizeNode($node) : Node {
+    public static function normalizeNode($node): Node {
         if ($node instanceof Builder) {
             return $node->getNode();
-        } elseif ($node instanceof Node) {
+        }
+
+        if ($node instanceof Node) {
             return $node;
         }
 
@@ -42,7 +44,7 @@ final class BuilderHelpers
      *
      * @return Stmt The normalized statement node
      */
-    public static function normalizeStmt($node) : Stmt {
+    public static function normalizeStmt($node): Stmt {
         $node = self::normalizeNode($node);
         if ($node instanceof Stmt) {
             return $node;
@@ -62,7 +64,7 @@ final class BuilderHelpers
      *
      * @return Identifier The normalized identifier
      */
-    public static function normalizeIdentifier($name) : Identifier {
+    public static function normalizeIdentifier($name): Identifier {
         if ($name instanceof Identifier) {
             return $name;
         }
@@ -100,8 +102,28 @@ final class BuilderHelpers
      *
      * @return Name The normalized name
      */
-    public static function normalizeName($name) : Name {
+    public static function normalizeName($name): Name {
-        return self::normalizeNameCommon($name, false);
+        if ($name instanceof Name) {
+            return $name;
+        }
+
+        if (is_string($name)) {
+            if (!$name) {
+                throw new \LogicException('Name cannot be empty');
+            }
+
+            if ($name[0] === '\\') {
+                return new Name\FullyQualified(substr($name, 1));
+            }
+
+            if (0 === strpos($name, 'namespace\\')) {
+                return new Name\Relative(substr($name, strlen('namespace\\')));
+            }
+
+            return new Name($name);
+        }
+
+        throw new \LogicException('Name must be a string or an instance of Node\Name');
     }
 
     /**
@@ -112,44 +134,17 @@ final class BuilderHelpers
      * @return Name|Expr The normalized name or expression
      */
     public static function normalizeNameOrExpr($name) {
-        return self::normalizeNameCommon($name, true);
+        if ($name instanceof Expr) {
-    }
-
-    /**
-     * Normalizes a name: Converts string names to Name nodes, optionally allowing expressions.
-     *
-     * @param Expr|Name|string $name      The name to normalize
-     * @param bool             $allowExpr Whether to also allow expressions
-     *
-     * @return Name|Expr The normalized name, or expression (if allowed)
-     */
-    private static function normalizeNameCommon($name, bool $allowExpr) {
-        if ($name instanceof Name) {
             return $name;
-        } elseif (is_string($name)) {
-            if (!$name) {
-                throw new \LogicException('Name cannot be empty');
-            }
-
-            if ($name[0] === '\\') {
-                return new Name\FullyQualified(substr($name, 1));
-            } elseif (0 === strpos($name, 'namespace\\')) {
-                return new Name\Relative(substr($name, strlen('namespace\\')));
-            } else {
-                return new Name($name);
-            }
         }
 
-        if ($allowExpr) {
+        if (!is_string($name) && !($name instanceof Name)) {
-            if ($name instanceof Expr) {
-                return $name;
-            }
             throw new \LogicException(
                 'Name must be a string or an instance of Node\Name or Node\Expr'
             );
-        } else {
-            throw new \LogicException('Name must be a string or an instance of Node\Name');
         }
+
+        return self::normalizeName($name);
     }
 
     /**
@@ -158,16 +153,19 @@ final class BuilderHelpers
      * In particular, builtin types become Identifiers, custom types become Names and nullables
      * are wrapped in NullableType nodes.
      *
-     * @param string|Name|Identifier|NullableType $type The type to normalize
+     * @param string|Name|Identifier|ComplexType $type The type to normalize
      *
-     * @return Name|Identifier|NullableType The normalized type
+     * @return Name|Identifier|ComplexType The normalized type
      */
     public static function normalizeType($type) {
         if (!is_string($type)) {
-            if (!$type instanceof Name && !$type instanceof Identifier
+            if (
-                    && !$type instanceof NullableType) {
+                !$type instanceof Name && !$type instanceof Identifier &&
+                !$type instanceof ComplexType
+            ) {
                 throw new \LogicException(
-                    'Type must be a string, or an instance of Name, Identifier or NullableType');
+                    'Type must be a string, or an instance of Name, Identifier or ComplexType'
+                );
             }
             return $type;
         }
@@ -179,7 +177,20 @@ final class BuilderHelpers
         }
 
         $builtinTypes = [
-            'array', 'callable', 'string', 'int', 'float', 'bool', 'iterable', 'void', 'object'
+            'array',
+            'callable',
+            'bool',
+            'int',
+            'float',
+            'string',
+            'iterable',
+            'void',
+            'object',
+            'null',
+            'false',
+            'mixed',
+            'never',
+            'true',
         ];
 
         $lowerType = strtolower($type);
@@ -189,11 +200,14 @@ final class BuilderHelpers
             $type = self::normalizeName($type);
         }
 
-        if ($nullable && (string) $type === 'void') {
+        $notNullableTypes = [
-            throw new \LogicException('void type cannot be nullable');
+            'void', 'mixed', 'never',
+        ];
+        if ($nullable && in_array((string) $type, $notNullableTypes)) {
+            throw new \LogicException(sprintf('%s type cannot be nullable', $type));
         }
 
-        return $nullable ? new Node\NullableType($type) : $type;
+        return $nullable ? new NullableType($type) : $type;
     }
 
     /**
@@ -204,35 +218,47 @@ final class BuilderHelpers
      *
      * @return Expr The normalized value
      */
-    public static function normalizeValue($value) : Expr {
+    public static function normalizeValue($value): Expr {
         if ($value instanceof Node\Expr) {
             return $value;
-        } elseif (is_null($value)) {
+        }
+
+        if (is_null($value)) {
             return new Expr\ConstFetch(
                 new Name('null')
             );
-        } elseif (is_bool($value)) {
+        }
+
+        if (is_bool($value)) {
             return new Expr\ConstFetch(
                 new Name($value ? 'true' : 'false')
             );
-        } elseif (is_int($value)) {
+        }
-            return new Scalar\LNumber($value);
+
-        } elseif (is_float($value)) {
+        if (is_int($value)) {
-            return new Scalar\DNumber($value);
+            return new Scalar\Int_($value);
-        } elseif (is_string($value)) {
+        }
+
+        if (is_float($value)) {
+            return new Scalar\Float_($value);
+        }
+
+        if (is_string($value)) {
             return new Scalar\String_($value);
-        } elseif (is_array($value)) {
+        }
+
+        if (is_array($value)) {
             $items = [];
             $lastKey = -1;
             foreach ($value as $itemKey => $itemValue) {
                 // for consecutive, numeric keys don't generate keys
                 if (null !== $lastKey && ++$lastKey === $itemKey) {
-                    $items[] = new Expr\ArrayItem(
+                    $items[] = new Node\ArrayItem(
                         self::normalizeValue($itemValue)
                     );
                 } else {
                     $lastKey = null;
-                    $items[] = new Expr\ArrayItem(
+                    $items[] = new Node\ArrayItem(
                         self::normalizeValue($itemValue),
                         self::normalizeValue($itemKey)
                     );
@@ -240,9 +266,9 @@ final class BuilderHelpers
             }
 
             return new Expr\Array_($items);
-        } else {
-            throw new \LogicException('Invalid value');
         }
+
+        throw new \LogicException('Invalid value');
     }
 
     /**
@@ -252,14 +278,35 @@ final class BuilderHelpers
      *
      * @return Comment\Doc The normalized doc comment
      */
-    public static function normalizeDocComment($docComment) : Comment\Doc {
+    public static function normalizeDocComment($docComment): Comment\Doc {
         if ($docComment instanceof Comment\Doc) {
             return $docComment;
-        } elseif (is_string($docComment)) {
-            return new Comment\Doc($docComment);
-        } else {
-            throw new \LogicException('Doc comment must be a string or an instance of PhpParser\Comment\Doc');
         }
+
+        if (is_string($docComment)) {
+            return new Comment\Doc($docComment);
+        }
+
+        throw new \LogicException('Doc comment must be a string or an instance of PhpParser\Comment\Doc');
+    }
+
+    /**
+     * Normalizes a attribute: Converts attribute to the Attribute Group if needed.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return Node\AttributeGroup The Attribute Group
+     */
+    public static function normalizeAttribute($attribute): Node\AttributeGroup {
+        if ($attribute instanceof Node\AttributeGroup) {
+            return $attribute;
+        }
+
+        if (!($attribute instanceof Node\Attribute)) {
+            throw new \LogicException('Attribute must be an instance of PhpParser\Node\Attribute or PhpParser\Node\AttributeGroup');
+        }
+
+        return new Node\AttributeGroup([$attribute]);
     }
 
     /**
@@ -270,8 +317,17 @@ final class BuilderHelpers
      *
      * @return int New modifiers
      */
-    public static function addModifier(int $modifiers, int $modifier) : int {
+    public static function addModifier(int $modifiers, int $modifier): int {
         Stmt\Class_::verifyModifier($modifiers, $modifier);
         return $modifiers | $modifier;
     }
+
+    /**
+     * Adds a modifier and returns new modifier bitmask.
+     * @return int New modifiers
+     */
+    public static function addClassModifier(int $existingModifiers, int $modifierToSet): int {
+        Stmt\Class_::verifyClassModifier($existingModifiers, $modifierToSet);
+        return $existingModifiers | $modifierToSet;
+    }
 }

lib/PhpParser/Comment.php

@@ -2,12 +2,14 @@
 
 namespace PhpParser;
 
-class Comment implements \JsonSerializable
+class Comment implements \JsonSerializable {
-{
     protected $text;
-    protected $line;
+    protected $startLine;
-    protected $filePos;
+    protected $startFilePos;
-    protected $tokenPos;
+    protected $startTokenPos;
+    protected $endLine;
+    protected $endFilePos;
+    protected $endTokenPos;
 
     /**
      * Constructs a comment node.
@@ -18,12 +20,17 @@ class Comment implements \JsonSerializable
      * @param int    $startTokenPos Token offset the comment started on
      */
     public function __construct(
-        string $text, int $startLine = -1, int $startFilePos = -1, int $startTokenPos = -1
+        string $text,
+        int $startLine = -1, int $startFilePos = -1, int $startTokenPos = -1,
+        int $endLine = -1, int $endFilePos = -1, int $endTokenPos = -1
     ) {
         $this->text = $text;
-        $this->line = $startLine;
+        $this->startLine = $startLine;
-        $this->filePos = $startFilePos;
+        $this->startFilePos = $startFilePos;
-        $this->tokenPos = $startTokenPos;
+        $this->startTokenPos = $startTokenPos;
+        $this->endLine = $endLine;
+        $this->endFilePos = $endFilePos;
+        $this->endTokenPos = $endTokenPos;
     }
 
     /**
@@ -31,35 +38,95 @@ class Comment implements \JsonSerializable
      *
      * @return string The comment text (including comment delimiters like /*)
      */
-    public function getText() : string {
+    public function getText(): string {
         return $this->text;
     }
 
     /**
      * Gets the line number the comment started on.
      *
-     * @return int Line number
+     * @return int Line number (or -1 if not available)
      */
-    public function getLine() : int {
+    public function getStartLine(): int {
-        return $this->line;
+        return $this->startLine;
     }
 
     /**
      * Gets the file offset the comment started on.
      *
-     * @return int File offset
+     * @return int File offset (or -1 if not available)
      */
-    public function getFilePos() : int {
+    public function getStartFilePos(): int {
-        return $this->filePos;
+        return $this->startFilePos;
     }
 
     /**
      * Gets the token offset the comment started on.
      *
+     * @return int Token offset (or -1 if not available)
+     */
+    public function getStartTokenPos(): int {
+        return $this->startTokenPos;
+    }
+
+    /**
+     * Gets the line number the comment ends on.
+     *
+     * @return int Line number (or -1 if not available)
+     */
+    public function getEndLine(): int {
+        return $this->endLine;
+    }
+
+    /**
+     * Gets the file offset the comment ends on.
+     *
+     * @return int File offset (or -1 if not available)
+     */
+    public function getEndFilePos(): int {
+        return $this->endFilePos;
+    }
+
+    /**
+     * Gets the token offset the comment ends on.
+     *
+     * @return int Token offset (or -1 if not available)
+     */
+    public function getEndTokenPos(): int {
+        return $this->endTokenPos;
+    }
+
+    /**
+     * Gets the line number the comment started on.
+     *
+     * @deprecated Use getStartLine() instead
+     *
+     * @return int Line number
+     */
+    public function getLine(): int {
+        return $this->startLine;
+    }
+
+    /**
+     * Gets the file offset the comment started on.
+     *
+     * @deprecated Use getStartFilePos() instead
+     *
+     * @return int File offset
+     */
+    public function getFilePos(): int {
+        return $this->startFilePos;
+    }
+
+    /**
+     * Gets the token offset the comment started on.
+     *
+     * @deprecated Use getStartTokenPos() instead
+     *
      * @return int Token offset
      */
-    public function getTokenPos() : int {
+    public function getTokenPos(): int {
-        return $this->tokenPos;
+        return $this->startTokenPos;
     }
 
     /**
@@ -67,7 +134,7 @@ class Comment implements \JsonSerializable
      *
      * @return string The comment text (including comment delimiters like /*)
      */
-    public function __toString() : string {
+    public function __toString(): string {
         return $this->text;
     }
 
@@ -82,12 +149,13 @@ class Comment implements \JsonSerializable
      * @return mixed|string
      */
     public function getReformattedText() {
-        $text = trim($this->text);
+        $text = $this->text;
         $newlinePos = strpos($text, "\n");
         if (false === $newlinePos) {
             // Single line comments don't need further processing
             return $text;
-        } elseif (preg_match('((*BSR_ANYCRLF)(*ANYCRLF)^.*(?:\R\s+\*.*)+$)', $text)) {
+        }
+        if (preg_match('((*BSR_ANYCRLF)(*ANYCRLF)^.*(?:\R\s+\*.*)+$)', $text)) {
             // Multi line comment of the type
             //
             //     /*
@@ -97,7 +165,8 @@ class Comment implements \JsonSerializable
             //
             // is handled by replacing the whitespace sequences before the * by a single space
             return preg_replace('(^\s+\*)m', ' *', $this->text);
-        } elseif (preg_match('(^/\*\*?\s*[\r\n])', $text) && preg_match('(\n(\s*)\*/$)', $text, $matches)) {
+        }
+        if (preg_match('(^/\*\*?\s*[\r\n])', $text) && preg_match('(\n(\s*)\*/$)', $text, $matches)) {
             // Multi line comment of the type
             //
             //    /*
@@ -109,7 +178,8 @@ class Comment implements \JsonSerializable
             // */ on all lines. So if the last line is "    */", then "    " is removed at the
             // start of all lines.
             return preg_replace('(^' . preg_quote($matches[1]) . ')m', '', $text);
-        } elseif (preg_match('(^/\*\*?\s*(?!\s))', $text, $matches)) {
+        }
+        if (preg_match('(^/\*\*?\s*(?!\s))', $text, $matches)) {
             // Multi line comment of the type
             //
             //     /* Some text.
@@ -136,7 +206,7 @@ class Comment implements \JsonSerializable
      * @param string $str String to check
      * @return int Length in characters. Tabs count as single characters.
      */
-    private function getShortestWhitespacePrefixLen(string $str) : int {
+    private function getShortestWhitespacePrefixLen(string $str): int {
         $lines = explode("\n", $str);
         $shortestPrefixLen = \INF;
         foreach ($lines as $line) {
@@ -153,15 +223,19 @@ class Comment implements \JsonSerializable
      * @return       array
      * @psalm-return array{nodeType:string, text:mixed, line:mixed, filePos:mixed}
      */
-    public function jsonSerialize() : array {
+    public function jsonSerialize(): array {
         // Technically not a node, but we make it look like one anyway
         $type = $this instanceof Comment\Doc ? 'Comment_Doc' : 'Comment';
         return [
             'nodeType' => $type,
             'text' => $this->text,
-            'line' => $this->line,
+            // TODO: Rename these to include "start".
-            'filePos' => $this->filePos,
+            'line' => $this->startLine,
-            'tokenPos' => $this->tokenPos,
+            'filePos' => $this->startFilePos,
+            'tokenPos' => $this->startTokenPos,
+            'endLine' => $this->endLine,
+            'endFilePos' => $this->endFilePos,
+            'endTokenPos' => $this->endTokenPos,
         ];
     }
 }

lib/PhpParser/Comment/Doc.php

@@ -2,6 +2,5 @@
 
 namespace PhpParser\Comment;
 
-class Doc extends \PhpParser\Comment
+class Doc extends \PhpParser\Comment {
-{
 }

lib/PhpParser/ConstExprEvaluationException.php

@@ -1,6 +1,6 @@
-<?php
+<?php declare(strict_types=1);
 
 namespace PhpParser;
 
-class ConstExprEvaluationException extends \Exception
+class ConstExprEvaluationException extends \Exception {
-{}
+}

lib/PhpParser/ConstExprEvaluator.php

@@ -1,10 +1,12 @@
-<?php
+<?php declare(strict_types=1);
 
 namespace PhpParser;
 
 use PhpParser\Node\Expr;
 use PhpParser\Node\Scalar;
 
+use function array_merge;
+
 /**
  * Evaluates constant expressions.
  *
@@ -24,8 +26,7 @@ use PhpParser\Node\Scalar;
  * point to string conversions are affected by the precision ini setting. Secondly, they are also
  * affected by the LC_NUMERIC locale.
  */
-class ConstExprEvaluator
+class ConstExprEvaluator {
-{
     private $fallbackEvaluator;
 
     /**
@@ -36,8 +37,8 @@ class ConstExprEvaluator
      *
      * @param callable|null $fallbackEvaluator To call if subexpression cannot be evaluated
      */
-    public function __construct(callable $fallbackEvaluator = null) {
+    public function __construct(?callable $fallbackEvaluator = null) {
-        $this->fallbackEvaluator = $fallbackEvaluator ?? function(Expr $expr) {
+        $this->fallbackEvaluator = $fallbackEvaluator ?? function (Expr $expr) {
             throw new ConstExprEvaluationException(
                 "Expression of type {$expr->getType()} cannot be evaluated"
             );
@@ -62,7 +63,7 @@ class ConstExprEvaluator
      * @throws ConstExprEvaluationException if the expression cannot be evaluated or an error occurred
      */
     public function evaluateSilently(Expr $expr) {
-        set_error_handler(function($num, $str, $file, $line) {
+        set_error_handler(function ($num, $str, $file, $line) {
             throw new \ErrorException($str, 0, $num, $file, $line);
         });
 
@@ -101,8 +102,8 @@ class ConstExprEvaluator
     }
 
     private function evaluate(Expr $expr) {
-        if ($expr instanceof Scalar\LNumber
+        if ($expr instanceof Scalar\Int_
-            || $expr instanceof Scalar\DNumber
+            || $expr instanceof Scalar\Float_
             || $expr instanceof Scalar\String_
         ) {
             return $expr->value;
@@ -150,6 +151,8 @@ class ConstExprEvaluator
         foreach ($expr->items as $item) {
             if (null !== $item->key) {
                 $array[$this->evaluate($item->key)] = $this->evaluate($item->value);
+            } elseif ($item->unpack) {
+                $array = array_merge($array, $this->evaluate($item->value));
             } else {
                 $array[] = $this->evaluate($item->value);
             }

lib/PhpParser/Error.php

@@ -2,8 +2,7 @@
 
 namespace PhpParser;
 
-class Error extends \RuntimeException
+class Error extends \RuntimeException {
-{
     protected $rawMessage;
     protected $attributes;
 
@@ -29,7 +28,7 @@ class Error extends \RuntimeException
      *
      * @return string Error message
      */
-    public function getRawMessage() : string {
+    public function getRawMessage(): string {
         return $this->rawMessage;
     }
 
@@ -38,7 +37,7 @@ class Error extends \RuntimeException
      *
      * @return int Error start line
      */
-    public function getStartLine() : int {
+    public function getStartLine(): int {
         return $this->attributes['startLine'] ?? -1;
     }
 
@@ -47,7 +46,7 @@ class Error extends \RuntimeException
      *
      * @return int Error end line
      */
-    public function getEndLine() : int {
+    public function getEndLine(): int {
         return $this->attributes['endLine'] ?? -1;
     }
 
@@ -56,7 +55,7 @@ class Error extends \RuntimeException
      *
      * @return array
      */
-    public function getAttributes() : array {
+    public function getAttributes(): array {
         return $this->attributes;
     }
 
@@ -97,7 +96,7 @@ class Error extends \RuntimeException
      *
      * @return bool
      */
-    public function hasColumnInfo() : bool {
+    public function hasColumnInfo(): bool {
         return isset($this->attributes['startFilePos'], $this->attributes['endFilePos']);
     }
 
@@ -107,7 +106,7 @@ class Error extends \RuntimeException
      * @param string $code Source code of the file
      * @return int
      */
-    public function getStartColumn(string $code) : int {
+    public function getStartColumn(string $code): int {
         if (!$this->hasColumnInfo()) {
             throw new \RuntimeException('Error does not have column information');
         }
@@ -121,7 +120,7 @@ class Error extends \RuntimeException
      * @param string $code Source code of the file
      * @return int
      */
-    public function getEndColumn(string $code) : int {
+    public function getEndColumn(string $code): int {
         if (!$this->hasColumnInfo()) {
             throw new \RuntimeException('Error does not have column information');
         }
@@ -136,7 +135,7 @@ class Error extends \RuntimeException
      *
      * @return string Formatted message
      */
-    public function getMessageWithColumnInfo(string $code) : string {
+    public function getMessageWithColumnInfo(string $code): string {
         return sprintf(
             '%s from %d:%d to %d:%d', $this->getRawMessage(),
             $this->getStartLine(), $this->getStartColumn($code),
@@ -152,7 +151,7 @@ class Error extends \RuntimeException
      *
      * @return int 1-based column (relative to start of line)
      */
-    private function toColumn(string $code, int $pos) : int {
+    private function toColumn(string $code, int $pos): int {
         if ($pos > strlen($code)) {
             throw new \RuntimeException('Invalid position information');
         }

lib/PhpParser/ErrorHandler.php

@@ -2,8 +2,7 @@
 
 namespace PhpParser;
 
-interface ErrorHandler
+interface ErrorHandler {
-{
     /**
      * Handle an error generated during lexing, parsing or some other operation.
      *

lib/PhpParser/ErrorHandler/Collecting.php

@@ -10,8 +10,7 @@ use PhpParser\ErrorHandler;
  *
  * This allows graceful handling of errors.
  */
-class Collecting implements ErrorHandler
+class Collecting implements ErrorHandler {
-{
     /** @var Error[] Collected errors */
     private $errors = [];
 
@@ -24,7 +23,7 @@ class Collecting implements ErrorHandler
      *
      * @return Error[]
      */
-    public function getErrors() : array {
+    public function getErrors(): array {
         return $this->errors;
     }
 
@@ -33,7 +32,7 @@ class Collecting implements ErrorHandler
      *
      * @return bool
      */
-    public function hasErrors() : bool {
+    public function hasErrors(): bool {
         return !empty($this->errors);
     }
 

lib/PhpParser/ErrorHandler/Throwing.php

@@ -10,8 +10,7 @@ use PhpParser\ErrorHandler;
  *
  * This is the default strategy used by all components.
  */
-class Throwing implements ErrorHandler
+class Throwing implements ErrorHandler {
-{
     public function handleError(Error $error) {
         throw $error;
     }

lib/PhpParser/Internal/DiffElem.php

@@ -5,12 +5,11 @@ namespace PhpParser\Internal;
 /**
  * @internal
  */
-class DiffElem
+class DiffElem {
-{
+    public const TYPE_KEEP = 0;
-    const TYPE_KEEP = 0;
+    public const TYPE_REMOVE = 1;
-    const TYPE_REMOVE = 1;
+    public const TYPE_ADD = 2;
-    const TYPE_ADD = 2;
+    public const TYPE_REPLACE = 3;
-    const TYPE_REPLACE = 3;
 
     /** @var int One of the TYPE_* constants */
     public $type;

lib/PhpParser/Internal/Differ.php

@@ -10,8 +10,7 @@ namespace PhpParser\Internal;
  *
  * @internal
  */
-class Differ
+class Differ {
-{
     private $isEqual;
 
     /**
@@ -31,7 +30,7 @@ class Differ
      *
      * @return DiffElem[] Diff (edit script)
      */
-    public function diff(array $old, array $new) {
+    public function diff(array $old, array $new): array {
         list($trace, $x, $y) = $this->calculateTrace($old, $new);
         return $this->extractDiff($trace, $x, $y, $old, $new);
     }
@@ -47,7 +46,7 @@ class Differ
      *
      * @return DiffElem[] Diff (edit script), including replace operations
      */
-    public function diffWithReplacements(array $old, array $new) {
+    public function diffWithReplacements(array $old, array $new): array {
         return $this->coalesceReplacements($this->diff($old, $new));
     }
 
@@ -125,7 +124,7 @@ class Differ
      * @param DiffElem[] $diff
      * @return DiffElem[]
      */
-    private function coalesceReplacements(array $diff) {
+    private function coalesceReplacements(array $diff): array {
         $newDiff = [];
         $c = \count($diff);
         for ($i = 0; $i < $c; $i++) {

lib/PhpParser/Internal/PrintableNewAnonClassNode.php

@@ -15,8 +15,9 @@ use PhpParser\Node\Expr;
  *
  * @internal
  */
-class PrintableNewAnonClassNode extends Expr
+class PrintableNewAnonClassNode extends Expr {
-{
+    /** @var Node\AttributeGroup[] PHP attribute groups */
+    public $attrGroups;
     /** @var Node\Arg[] Arguments */
     public $args;
     /** @var null|Node\Name Name of extended class */
@@ -27,9 +28,11 @@ class PrintableNewAnonClassNode extends Expr
     public $stmts;
 
     public function __construct(
-        array $args, Node\Name $extends = null, array $implements, array $stmts, array $attributes
+        array $attrGroups, array $args, ?Node\Name $extends, array $implements,
+        array $stmts, array $attributes
     ) {
         parent::__construct($attributes);
+        $this->attrGroups = $attrGroups;
         $this->args = $args;
         $this->extends = $extends;
         $this->implements = $implements;
@@ -42,16 +45,16 @@ class PrintableNewAnonClassNode extends Expr
         // We don't assert that $class->name is null here, to allow consumers to assign unique names
         // to anonymous classes for their own purposes. We simplify ignore the name here.
         return new self(
-            $newNode->args, $class->extends, $class->implements,
+            $class->attrGroups, $newNode->args, $class->extends, $class->implements,
             $class->stmts, $newNode->getAttributes()
         );
     }
 
-    public function getType() : string {
+    public function getType(): string {
         return 'Expr_PrintableNewAnonClass';
     }
 
-    public function getSubNodeNames() : array {
+    public function getSubNodeNames(): array {
-        return ['args', 'extends', 'implements', 'stmts'];
+        return ['attrGroups', 'args', 'extends', 'implements', 'stmts'];
     }
 }

lib/PhpParser/Internal/TokenPolyfill.php

@@ -0,0 +1,272 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Internal;
+
+if (\PHP_VERSION_ID >= 80000) {
+    class TokenPolyfill extends \PhpToken {
+    }
+    return;
+}
+
+/**
+ * This is a polyfill for the PhpToken class introduced in PHP 8.0. We do not actually polyfill
+ * PhpToken, because composer might end up picking a different polyfill implementation, which does
+ * not meet our requirements.
+ *
+ * @internal
+ */
+class TokenPolyfill {
+    /** @var int The ID of the token. Either a T_* constant of a character code < 256. */
+    public $id;
+    /** @var string The textual content of the token. */
+    public $text;
+    /** @var int The 1-based starting line of the token (or -1 if unknown). */
+    public $line;
+    /** @var int The 0-based starting position of the token (or -1 if unknown). */
+    public $pos;
+
+    /** @var bool[] Tokens ignored by the PHP parser. */
+    private const IGNORABLE_TOKENS = [
+        \T_WHITESPACE => true,
+        \T_COMMENT => true,
+        \T_DOC_COMMENT => true,
+        \T_OPEN_TAG => true,
+    ];
+
+    /** @var bool[] Tokens that may be part of a T_NAME_* identifier. */
+    private static $identifierTokens;
+
+    /**
+     * Create a Token with the given ID and text, as well optional line and position information.
+     */
+    final public function __construct(int $id, string $text, int $line = -1, int $pos = -1) {
+        $this->id = $id;
+        $this->text = $text;
+        $this->line = $line;
+        $this->pos = $pos;
+    }
+
+    /**
+     * Get the name of the token. For single-char tokens this will be the token character.
+     * Otherwise it will be a T_* style name, or null if the token ID is unknown.
+     */
+    public function getTokenName(): ?string {
+        if ($this->id < 256) {
+            return \chr($this->id);
+        }
+
+        $name = token_name($this->id);
+        return $name === 'UNKNOWN' ? null : $name;
+    }
+
+    /**
+     * Check whether the token is of the given kind. The kind may be either an integer that matches
+     * the token ID, a string that matches the token text, or an array of integers/strings. In the
+     * latter case, the function returns true if any of the kinds in the array match.
+     *
+     * @param int|string|array $kind
+     */
+    public function is($kind): bool {
+        if (\is_int($kind)) {
+            return $this->id === $kind;
+        }
+        if (\is_string($kind)) {
+            return $this->text === $kind;
+        }
+        if (\is_array($kind)) {
+            foreach ($kind as $entry) {
+                if (\is_int($entry)) {
+                    if ($this->id === $entry) {
+                        return true;
+                    }
+                } elseif (\is_string($entry)) {
+                    if ($this->text === $entry) {
+                        return true;
+                    }
+                } else {
+                    throw new \TypeError(
+                        'Argument #1 ($kind) must only have elements of type string|int, ' .
+                        gettype($entry) . ' given');
+                }
+            }
+            return false;
+        }
+        throw new \TypeError(
+            'Argument #1 ($kind) must be of type string|int|array, ' .gettype($kind) . ' given');
+    }
+
+    /**
+     * Check whether this token would be ignored by the PHP parser. Returns true for T_WHITESPACE,
+     * T_COMMENT, T_DOC_COMMENT and T_OPEN_TAG, and false for everything else.
+     */
+    public function isIgnorable(): bool {
+        return isset(self::IGNORABLE_TOKENS[$this->id]);
+    }
+
+    /**
+     * Return the textual content of the token.
+     */
+    public function __toString(): string {
+        return $this->text;
+    }
+
+    /**
+     * Tokenize the given source code and return an array of tokens.
+     *
+     * This performs certain canonicalizations to match the PHP 8.0 token format:
+     *  * Bad characters are represented using T_BAD_CHARACTER rather than omitted.
+     *  * T_COMMENT does not include trailing newlines, instead the newline is part of a following
+     *    T_WHITESPACE token.
+     *  * Namespaced names are represented using T_NAME_* tokens.
+     *
+     * @returns static[]
+     */
+    public static function tokenize(string $code, int $flags = 0): array {
+        self::init();
+
+        $tokens = [];
+        $line = 1;
+        $pos = 0;
+        $origTokens = \token_get_all($code, $flags);
+        if (\PHP_VERSION_ID < 70400) {
+            $origTokens = self::fixupBadCharacters($code, $origTokens);
+        }
+
+        $numTokens = \count($origTokens);
+        for ($i = 0; $i < $numTokens; $i++) {
+            $token = $origTokens[$i];
+            if (\is_string($token)) {
+                if (\strlen($token) === 2) {
+                    // b" and B" are tokenized as single-char tokens, even though they aren't.
+                    $tokens[] = new static(\ord('"'), $token, $line, $pos);
+                    $pos += 2;
+                } else {
+                    $tokens[] = new static(\ord($token), $token, $line, $pos);
+                    $pos++;
+                }
+            } else {
+                $id = $token[0];
+                $text = $token[1];
+
+                // Emulate PHP 8.0 comment format, which does not include trailing whitespace anymore.
+                if ($id === \T_COMMENT && \substr($text, 0, 2) !== '/*' &&
+                    \preg_match('/(\r\n|\n|\r)$/D', $text, $matches)
+                ) {
+                    $trailingNewline = $matches[0];
+                    $text = \substr($text, 0, -\strlen($trailingNewline));
+                    $tokens[] = new static($id, $text, $line, $pos);
+                    $pos += \strlen($text);
+
+                    if ($i + 1 < $numTokens && $origTokens[$i + 1][0] === \T_WHITESPACE) {
+                        // Move trailing newline into following T_WHITESPACE token, if it already exists.
+                        $origTokens[$i + 1][1] = $trailingNewline . $origTokens[$i + 1][1];
+                        $origTokens[$i + 1][2]--;
+                    } else {
+                        // Otherwise, we need to create a new T_WHITESPACE token.
+                        $tokens[] = new static(\T_WHITESPACE, $trailingNewline, $line, $pos);
+                        $line++;
+                        $pos += \strlen($trailingNewline);
+                    }
+                    continue;
+                }
+
+                // Emulate PHP 8.0 T_NAME_* tokens, by combining sequences of T_NS_SEPARATOR and
+                // T_STRING into a single token.
+                if (($id === \T_NS_SEPARATOR || isset(self::$identifierTokens[$id]))) {
+                    $newText = $text;
+                    $lastWasSeparator = $id === \T_NS_SEPARATOR;
+                    for ($j = $i + 1; $j < $numTokens; $j++) {
+                        if ($lastWasSeparator) {
+                            if (!isset(self::$identifierTokens[$origTokens[$j][0]])) {
+                                break;
+                            }
+                            $lastWasSeparator = false;
+                        } else {
+                            if ($origTokens[$j][0] !== \T_NS_SEPARATOR) {
+                                break;
+                            }
+                            $lastWasSeparator = true;
+                        }
+                        $newText .= $origTokens[$j][1];
+                    }
+                    if ($lastWasSeparator) {
+                        // Trailing separator is not part of the name.
+                        $j--;
+                        $newText = \substr($newText, 0, -1);
+                    }
+                    if ($j > $i + 1) {
+                        if ($id === \T_NS_SEPARATOR) {
+                            $id = \T_NAME_FULLY_QUALIFIED;
+                        } elseif ($id === \T_NAMESPACE) {
+                            $id = \T_NAME_RELATIVE;
+                        } else {
+                            $id = \T_NAME_QUALIFIED;
+                        }
+                        $tokens[] = new static($id, $newText, $line, $pos);
+                        $pos += \strlen($newText);
+                        $i = $j - 1;
+                        continue;
+                    }
+                }
+
+                $tokens[] = new static($id, $text, $line, $pos);
+                $line += \substr_count($text, "\n");
+                $pos += \strlen($text);
+            }
+        }
+        return $tokens;
+    }
+
+    /**
+     * Prior to PHP 7.4, token_get_all() simply dropped invalid characters from the token stream.
+     * Detect such cases and replace them with T_BAD_CHARACTER.
+     */
+    private static function fixupBadCharacters(string $code, array $origTokens): array {
+        $newTokens = [];
+        $pos = 0;
+        foreach ($origTokens as $token) {
+            $text = \is_string($token) ? $token : $token[1];
+            $len = \strlen($text);
+            if (substr($code, $pos, $len) !== $text) {
+                $nextPos = strpos($code, $text, $pos);
+                for ($i = $pos; $i < $nextPos; $i++) {
+                    // Don't bother including the line, we're not going to use it anyway.
+                    $newTokens[] = [\T_BAD_CHARACTER, $code[$i]];
+                }
+                $pos = $nextPos;
+            }
+            $pos += $len;
+            $newTokens[] = $token;
+        }
+
+        // Handle trailing invalid characters.
+        $codeLen = \strlen($code);
+        if ($pos !== $codeLen) {
+            for ($i = $pos; $i < $codeLen; $i++) {
+                $newTokens[] = [\T_BAD_CHARACTER, $code[$i]];
+            }
+        }
+        return $newTokens;
+    }
+
+    /** Initialize private static state needed by tokenize(). */
+    private static function init(): void {
+        if (isset(self::$identifierTokens)) {
+            return;
+        }
+
+        // Based on semi_reserved production.
+        self::$identifierTokens = \array_fill_keys([
+            \T_STRING,
+            \T_STATIC, \T_ABSTRACT, \T_FINAL, \T_PRIVATE, \T_PROTECTED, \T_PUBLIC, \T_READONLY,
+            \T_INCLUDE, \T_INCLUDE_ONCE, \T_EVAL, \T_REQUIRE, \T_REQUIRE_ONCE, \T_LOGICAL_OR, \T_LOGICAL_XOR, \T_LOGICAL_AND,
+            \T_INSTANCEOF, \T_NEW, \T_CLONE, \T_EXIT, \T_IF, \T_ELSEIF, \T_ELSE, \T_ENDIF, \T_ECHO, \T_DO, \T_WHILE,
+            \T_ENDWHILE, \T_FOR, \T_ENDFOR, \T_FOREACH, \T_ENDFOREACH, \T_DECLARE, \T_ENDDECLARE, \T_AS, \T_TRY, \T_CATCH,
+            \T_FINALLY, \T_THROW, \T_USE, \T_INSTEADOF, \T_GLOBAL, \T_VAR, \T_UNSET, \T_ISSET, \T_EMPTY, \T_CONTINUE, \T_GOTO,
+            \T_FUNCTION, \T_CONST, \T_RETURN, \T_PRINT, \T_YIELD, \T_LIST, \T_SWITCH, \T_ENDSWITCH, \T_CASE, \T_DEFAULT,
+            \T_BREAK, \T_ARRAY, \T_CALLABLE, \T_EXTENDS, \T_IMPLEMENTS, \T_NAMESPACE, \T_TRAIT, \T_INTERFACE, \T_CLASS,
+            \T_CLASS_C, \T_TRAIT_C, \T_FUNC_C, \T_METHOD_C, \T_LINE, \T_FILE, \T_DIR, \T_NS_C, \T_HALT_COMPILER, \T_FN,
+            \T_MATCH,
+        ], true);
+    }
+}

lib/PhpParser/Internal/TokenStream.php

@@ -2,14 +2,15 @@
 
 namespace PhpParser\Internal;
 
+use PhpParser\Token;
+
 /**
  * Provides operations on token streams, for use by pretty printer.
  *
  * @internal
  */
-class TokenStream
+class TokenStream {
-{
+    /** @var Token[] Tokens (in PhpToken::tokenize() format) */
-    /** @var array Tokens (in token_get_all format) */
     private $tokens;
     /** @var int[] Map from position to indentation */
     private $indentMap;
@@ -17,7 +18,7 @@ class TokenStream
     /**
      * Create token stream instance.
      *
-     * @param array $tokens Tokens in token_get_all() format
+     * @param Token[] $tokens Tokens in PhpToken::tokenize() format
      */
     public function __construct(array $tokens) {
         $this->tokens = $tokens;
@@ -32,8 +33,8 @@ class TokenStream
      *
      * @return bool
      */
-    public function haveParens(int $startPos, int $endPos) : bool {
+    public function haveParens(int $startPos, int $endPos): bool {
-        return $this->haveTokenImmediativelyBefore($startPos, '(')
+        return $this->haveTokenImmediatelyBefore($startPos, '(')
             && $this->haveTokenImmediatelyAfter($endPos, ')');
     }
 
@@ -45,8 +46,9 @@ class TokenStream
      *
      * @return bool
      */
-    public function haveBraces(int $startPos, int $endPos) : bool {
+    public function haveBraces(int $startPos, int $endPos): bool {
-        return $this->haveTokenImmediativelyBefore($startPos, '{')
+        return ($this->haveTokenImmediatelyBefore($startPos, '{')
+                || $this->haveTokenImmediatelyBefore($startPos, T_CURLY_OPEN))
             && $this->haveTokenImmediatelyAfter($endPos, '}');
     }
 
@@ -60,16 +62,15 @@ class TokenStream
      *
      * @return bool Whether the expected token was found
      */
-    public function haveTokenImmediativelyBefore(int $pos, $expectedTokenType) : bool {
+    public function haveTokenImmediatelyBefore(int $pos, $expectedTokenType): bool {
         $tokens = $this->tokens;
         $pos--;
         for (; $pos >= 0; $pos--) {
-            $tokenType = $tokens[$pos][0];
+            $token = $tokens[$pos];
-            if ($tokenType === $expectedTokenType) {
+            if ($token->is($expectedTokenType)) {
                 return true;
             }
-            if ($tokenType !== \T_WHITESPACE
+            if (!$token->isIgnorable()) {
-                && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
                 break;
             }
         }
@@ -86,16 +87,15 @@ class TokenStream
      *
      * @return bool Whether the expected token was found
      */
-    public function haveTokenImmediatelyAfter(int $pos, $expectedTokenType) : bool {
+    public function haveTokenImmediatelyAfter(int $pos, $expectedTokenType): bool {
         $tokens = $this->tokens;
         $pos++;
-        for (; $pos < \count($tokens); $pos++) {
+        for ($c = \count($tokens); $pos < $c; $pos++) {
-            $tokenType = $tokens[$pos][0];
+            $token = $tokens[$pos];
-            if ($tokenType === $expectedTokenType) {
+            if ($token->is($expectedTokenType)) {
                 return true;
             }
-            if ($tokenType !== \T_WHITESPACE
+            if (!$token->isIgnorable()) {
-                && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
                 break;
             }
         }
@@ -110,7 +110,7 @@ class TokenStream
             return $pos;
         }
 
-        if ($tokens[$pos][0] !== $skipTokenType) {
+        if (!$tokens[$pos]->is($skipTokenType)) {
             // Shouldn't happen. The skip token MUST be there
             throw new \Exception('Encountered unexpected token');
         }
@@ -127,7 +127,7 @@ class TokenStream
             return $pos;
         }
 
-        if ($tokens[$pos][0] !== $skipTokenType) {
+        if (!$tokens[$pos]->is($skipTokenType)) {
             // Shouldn't happen. The skip token MUST be there
             throw new \Exception('Encountered unexpected token');
         }
@@ -142,11 +142,10 @@ class TokenStream
      * @param int $pos Token position
      * @return int Non-whitespace token position
      */
-    public function skipLeftWhitespace(int $pos) {
+    public function skipLeftWhitespace(int $pos): int {
         $tokens = $this->tokens;
         for (; $pos >= 0; $pos--) {
-            $type = $tokens[$pos][0];
+            if (!$tokens[$pos]->isIgnorable()) {
-            if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
         }
@@ -159,28 +158,50 @@ class TokenStream
      * @param int $pos Token position
      * @return int Non-whitespace token position
      */
-    public function skipRightWhitespace(int $pos) {
+    public function skipRightWhitespace(int $pos): int {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
-            $type = $tokens[$pos][0];
+            if (!$tokens[$pos]->isIgnorable()) {
-            if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
         }
         return $pos;
     }
 
-    public function findRight($pos, $findTokenType) {
+    public function findRight(int $pos, $findTokenType) {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
-            $type = $tokens[$pos][0];
+            if ($tokens[$pos]->is($findTokenType)) {
-            if ($type === $findTokenType) {
                 return $pos;
             }
         }
         return -1;
     }
 
+    /**
+     * Whether the given position range contains a certain token type.
+     *
+     * @param int $startPos Starting position (inclusive)
+     * @param int $endPos Ending position (exclusive)
+     * @param int|string $tokenType Token type to look for
+     * @return bool Whether the token occurs in the given range
+     */
+    public function haveTokenInRange(int $startPos, int $endPos, $tokenType): bool {
+        $tokens = $this->tokens;
+        for ($pos = $startPos; $pos < $endPos; $pos++) {
+            if ($tokens[$pos]->is($tokenType)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public function haveBracesInRange(int $startPos, int $endPos) {
+        return $this->haveTokenInRange($startPos, $endPos, '{')
+            || $this->haveTokenInRange($startPos, $endPos, T_CURLY_OPEN)
+            || $this->haveTokenInRange($startPos, $endPos, '}');
+    }
+
     /**
      * Get indentation before token position.
      *
@@ -188,7 +209,7 @@ class TokenStream
      *
      * @return int Indentation depth (in spaces)
      */
-    public function getIndentationBefore(int $pos) : int {
+    public function getIndentationBefore(int $pos): int {
         return $this->indentMap[$pos];
     }
 
@@ -201,28 +222,24 @@ class TokenStream
      *
      * @return string Code corresponding to token range, adjusted for indentation
      */
-    public function getTokenCode(int $from, int $to, int $indent) : string {
+    public function getTokenCode(int $from, int $to, int $indent): string {
         $tokens = $this->tokens;
         $result = '';
         for ($pos = $from; $pos < $to; $pos++) {
             $token = $tokens[$pos];
-            if (\is_array($token)) {
+            $id = $token->id;
-                $type = $token[0];
+            $text = $token->text;
-                $content = $token[1];
+            if ($id === \T_CONSTANT_ENCAPSED_STRING || $id === \T_ENCAPSED_AND_WHITESPACE) {
-                if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
+                $result .= $text;
-                    $result .= $content;
-                } else {
-                    // TODO Handle non-space indentation
-                    if ($indent < 0) {
-                        $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
-                    } elseif ($indent > 0) {
-                        $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
-                    } else {
-                        $result .= $content;
-                    }
-                }
             } else {
-                $result .= $token;
+                // TODO Handle non-space indentation
+                if ($indent < 0) {
+                    $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $text);
+                } elseif ($indent > 0) {
+                    $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $text);
+                } else {
+                    $result .= $text;
+                }
             }
         }
         return $result;
@@ -233,14 +250,14 @@ class TokenStream
      *
      * @return int[] Token position to indentation map
      */
-    private function calcIndentMap() {
+    private function calcIndentMap(): array {
         $indentMap = [];
         $indent = 0;
         foreach ($this->tokens as $token) {
             $indentMap[] = $indent;
 
-            if ($token[0] === \T_WHITESPACE) {
+            if ($token->id === \T_WHITESPACE) {
-                $content = $token[1];
+                $content = $token->text;
                 $newlinePos = \strrpos($content, "\n");
                 if (false !== $newlinePos) {
                     $indent = \strlen($content) - $newlinePos - 1;

lib/PhpParser/JsonDecoder.php

@@ -2,8 +2,7 @@
 
 namespace PhpParser;
 
-class JsonDecoder
+class JsonDecoder {
-{
     /** @var \ReflectionClass[] Node type to reflection class map */
     private $reflectionClassCache;
 
@@ -29,7 +28,7 @@ class JsonDecoder
         return $value;
     }
 
-    private function decodeArray(array $array) : array {
+    private function decodeArray(array $array): array {
         $decodedArray = [];
         foreach ($array as $key => $value) {
             $decodedArray[$key] = $this->decodeRecursive($value);
@@ -37,7 +36,7 @@ class JsonDecoder
         return $decodedArray;
     }
 
-    private function decodeNode(array $value) : Node {
+    private function decodeNode(array $value): Node {
         $nodeType = $value['nodeType'];
         if (!\is_string($nodeType)) {
             throw new \RuntimeException('Node type must be a string');
@@ -66,18 +65,20 @@ class JsonDecoder
         return $node;
     }
 
-    private function decodeComment(array $value) : Comment {
+    private function decodeComment(array $value): Comment {
         $className = $value['nodeType'] === 'Comment' ? Comment::class : Comment\Doc::class;
         if (!isset($value['text'])) {
             throw new \RuntimeException('Comment must have text');
         }
 
         return new $className(
-            $value['text'], $value['line'] ?? -1, $value['filePos'] ?? -1, $value['tokenPos'] ?? -1
+            $value['text'],
+            $value['line'] ?? -1, $value['filePos'] ?? -1, $value['tokenPos'] ?? -1,
+            $value['endLine'] ?? -1, $value['endFilePos'] ?? -1, $value['endTokenPos'] ?? -1
         );
     }
 
-    private function reflectionClassFromNodeType(string $nodeType) : \ReflectionClass {
+    private function reflectionClassFromNodeType(string $nodeType): \ReflectionClass {
         if (!isset($this->reflectionClassCache[$nodeType])) {
             $className = $this->classNameFromNodeType($nodeType);
             $this->reflectionClassCache[$nodeType] = new \ReflectionClass($className);
@@ -85,7 +86,7 @@ class JsonDecoder
         return $this->reflectionClassCache[$nodeType];
     }
 
-    private function classNameFromNodeType(string $nodeType) : string {
+    private function classNameFromNodeType(string $nodeType): string {
         $className = 'PhpParser\\Node\\' . strtr($nodeType, '_', '\\');
         if (class_exists($className)) {
             return $className;

lib/PhpParser/Lexer.php

@@ -4,19 +4,26 @@ namespace PhpParser;
 
 use PhpParser\Parser\Tokens;
 
-class Lexer
+require __DIR__ . '/compatibility_tokens.php';
-{
+
+class Lexer {
+    /** @var string Code being tokenized */
     protected $code;
+    /** @var Token[] Array of tokens */
     protected $tokens;
+    /** @var int Current position in the token array */
     protected $pos;
-    protected $line;
-    protected $filePos;
     protected $prevCloseTagHasNewline;
 
-    protected $tokenMap;
     protected $dropTokens;
 
-    protected $usedAttributes;
+    private $attributeStartLineUsed;
+    private $attributeEndLineUsed;
+    private $attributeStartTokenPosUsed;
+    private $attributeEndTokenPosUsed;
+    private $attributeStartFilePosUsed;
+    private $attributeEndFilePosUsed;
+    private $attributeCommentsUsed;
 
     /**
      * Creates a Lexer.
@@ -28,21 +35,23 @@ class Lexer
      *                       first three. For more info see getNextToken() docs.
      */
     public function __construct(array $options = []) {
-        // map from internal tokens to PhpParser tokens
-        $this->tokenMap = $this->createTokenMap();
-
         // map of tokens to drop while lexing (the map is only used for isset lookup,
         // that's why the value is simply set to 1; the value is never actually used.)
         $this->dropTokens = array_fill_keys(
-            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT], 1
+            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
         );
 
-        // the usedAttributes member is a map of the used attribute names to a dummy
+        $defaultAttributes = ['comments', 'startLine', 'endLine'];
-        // value (here "true")
+        $usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
-        $options += [
+
-            'usedAttributes' => ['comments', 'startLine', 'endLine'],
+        // Create individual boolean properties to make these checks faster.
-        ];
+        $this->attributeStartLineUsed = isset($usedAttributes['startLine']);
-        $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
+        $this->attributeEndLineUsed = isset($usedAttributes['endLine']);
+        $this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
+        $this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
+        $this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
+        $this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
+        $this->attributeCommentsUsed = isset($usedAttributes['comments']);
     }
 
     /**
@@ -55,15 +64,13 @@ class Lexer
      * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
      *                                        ErrorHandler\Throwing
      */
-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+    public function startLexing(string $code, ?ErrorHandler $errorHandler = null) {
         if (null === $errorHandler) {
             $errorHandler = new ErrorHandler\Throwing();
         }
 
         $this->code = $code; // keep the code around for __halt_compiler() handling
-        $this->pos  = -1;
+        $this->pos = -1;
-        $this->line =  1;
-        $this->filePos = 0;
 
         // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
         // This ensures proper composability, because having a newline is the "safe" assumption.
@@ -71,129 +78,85 @@ class Lexer
 
         $scream = ini_set('xdebug.scream', '0');
 
-        error_clear_last();
+        $this->tokens = @Token::tokenize($code);
-        $this->tokens = @token_get_all($code);
+        $this->postprocessTokens($errorHandler);
-        $this->handleErrors($errorHandler);
 
         if (false !== $scream) {
             ini_set('xdebug.scream', $scream);
         }
     }
 
-    private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
+    private function handleInvalidCharacter(Token $token, ErrorHandler $errorHandler): void {
-        for ($i = $start; $i < $end; $i++) {
+        $chr = $token->text;
-            $chr = $this->code[$i];
+        if ($chr === "\0") {
-            if ($chr === 'b' || $chr === 'B') {
+            // PHP cuts error message after null byte, so need special case
-                // HHVM does not treat b" tokens correctly, so ignore these
+            $errorMsg = 'Unexpected null byte';
-                continue;
+        } else {
+            $errorMsg = sprintf(
+                'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
+            );
+        }
+
+        $errorHandler->handleError(new Error($errorMsg, [
+            'startLine' => $token->line,
+            'endLine' => $token->line,
+            'startFilePos' => $token->pos,
+            'endFilePos' => $token->pos,
+        ]));
+    }
+
+    private function isUnterminatedComment(Token $token): bool {
+        return $token->is([\T_COMMENT, \T_DOC_COMMENT])
+            && substr($token->text, 0, 2) === '/*'
+            && substr($token->text, -2) !== '*/';
+    }
+
+    protected function postprocessTokens(ErrorHandler $errorHandler) {
+        // This function reports errors (bad characters and unterminated comments) in the token
+        // array, and performs certain canonicalizations:
+        //  * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
+        //    T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
+        //  * Add a sentinel token with ID 0.
+
+        $numTokens = \count($this->tokens);
+        if ($numTokens === 0) {
+            // Empty input edge case: Just add the sentinel token.
+            $this->tokens[] = new Token(0, "\0", 1, 0);
+            return;
+        }
+
+        for ($i = 0; $i < $numTokens; $i++) {
+            $token = $this->tokens[$i];
+            if ($token->id === \T_BAD_CHARACTER) {
+                $this->handleInvalidCharacter($token, $errorHandler);
             }
 
-            if ($chr === "\0") {
+            if ($token->id === \ord('&')) {
-                // PHP cuts error message after null byte, so need special case
+                $next = $i + 1;
-                $errorMsg = 'Unexpected null byte';
+                while (isset($this->tokens[$next]) && $this->tokens[$next]->id === \T_WHITESPACE) {
-            } else {
+                    $next++;
-                $errorMsg = sprintf(
+                }
-                    'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
+                $followedByVarOrVarArg = isset($this->tokens[$next]) &&
-                );
+                    $this->tokens[$next]->is([\T_VARIABLE, \T_ELLIPSIS]);
+                $token->id = $followedByVarOrVarArg
+                    ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
+                    : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
             }
+        }
 
-            $errorHandler->handleError(new Error($errorMsg, [
+        // Check for unterminated comment
-                'startLine' => $line,
+        $lastToken = $this->tokens[$numTokens - 1];
-                'endLine' => $line,
+        if ($this->isUnterminatedComment($lastToken)) {
-                'startFilePos' => $i,
+            $errorHandler->handleError(new Error('Unterminated comment', [
-                'endFilePos' => $i,
+                'startLine' => $lastToken->line,
+                'endLine' => $lastToken->getEndLine(),
+                'startFilePos' => $lastToken->pos,
+                'endFilePos' => $lastToken->getEndPos(),
             ]));
         }
-    }
 
-    /**
+        // Add sentinel token.
-     * Check whether comment token is unterminated.
+        $this->tokens[] = new Token(0, "\0", $lastToken->getEndLine(), $lastToken->getEndPos());
-     *
-     * @return bool
-     */
-    private function isUnterminatedComment($token) : bool {
-        return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
-            && substr($token[1], 0, 2) === '/*'
-            && substr($token[1], -2) !== '*/';
-    }
-
-    /**
-     * Check whether an error *may* have occurred during tokenization.
-     *
-     * @return bool
-     */
-    private function errorMayHaveOccurred() : bool {
-        if (defined('HHVM_VERSION')) {
-            // In HHVM token_get_all() does not throw warnings, so we need to conservatively
-            // assume that an error occurred
-            return true;
-        }
-
-        return null !== error_get_last();
-    }
-
-    protected function handleErrors(ErrorHandler $errorHandler) {
-        if (!$this->errorMayHaveOccurred()) {
-            return;
-        }
-
-        // PHP's error handling for token_get_all() is rather bad, so if we want detailed
-        // error information we need to compute it ourselves. Invalid character errors are
-        // detected by finding "gaps" in the token array. Unterminated comments are detected
-        // by checking if a trailing comment has a "*/" at the end.
-
-        $filePos = 0;
-        $line = 1;
-        foreach ($this->tokens as $token) {
-            $tokenValue = \is_string($token) ? $token : $token[1];
-            $tokenLen = \strlen($tokenValue);
-
-            if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
-                // Something is missing, must be an invalid character
-                $nextFilePos = strpos($this->code, $tokenValue, $filePos);
-                $this->handleInvalidCharacterRange(
-                    $filePos, $nextFilePos, $line, $errorHandler);
-                $filePos = (int) $nextFilePos;
-            }
-
-            $filePos += $tokenLen;
-            $line += substr_count($tokenValue, "\n");
-        }
-
-        if ($filePos !== \strlen($this->code)) {
-            if (substr($this->code, $filePos, 2) === '/*') {
-                // Unlike PHP, HHVM will drop unterminated comments entirely
-                $comment = substr($this->code, $filePos);
-                $errorHandler->handleError(new Error('Unterminated comment', [
-                    'startLine' => $line,
-                    'endLine' => $line + substr_count($comment, "\n"),
-                    'startFilePos' => $filePos,
-                    'endFilePos' => $filePos + \strlen($comment),
-                ]));
-
-                // Emulate the PHP behavior
-                $isDocComment = isset($comment[3]) && $comment[3] === '*';
-                $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
-            } else {
-                // Invalid characters at the end of the input
-                $this->handleInvalidCharacterRange(
-                    $filePos, \strlen($this->code), $line, $errorHandler);
-            }
-            return;
-        }
-
-        if (count($this->tokens) > 0) {
-            // Check for unterminated comment
-            $lastToken = $this->tokens[count($this->tokens) - 1];
-            if ($this->isUnterminatedComment($lastToken)) {
-                $errorHandler->handleError(new Error('Unterminated comment', [
-                    'startLine' => $line - substr_count($lastToken[1], "\n"),
-                    'endLine' => $line,
-                    'startFilePos' => $filePos - \strlen($lastToken[1]),
-                    'endFilePos' => $filePos,
-                ]));
-            }
-        }
     }
 
     /**
@@ -218,91 +181,76 @@ class Lexer
      *
      * @return int Token id
      */
-    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
+    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null): int {
         $startAttributes = [];
         $endAttributes   = [];
 
         while (1) {
-            if (isset($this->tokens[++$this->pos])) {
+            $token = $this->tokens[++$this->pos];
-                $token = $this->tokens[$this->pos];
-            } else {
-                // EOF token with ID 0
-                $token = "\0";
-            }
 
-            if (isset($this->usedAttributes['startLine'])) {
+            if ($this->attributeStartLineUsed) {
-                $startAttributes['startLine'] = $this->line;
+                $startAttributes['startLine'] = $token->line;
             }
-            if (isset($this->usedAttributes['startTokenPos'])) {
+            if ($this->attributeStartTokenPosUsed) {
                 $startAttributes['startTokenPos'] = $this->pos;
             }
-            if (isset($this->usedAttributes['startFilePos'])) {
+            if ($this->attributeStartFilePosUsed) {
-                $startAttributes['startFilePos'] = $this->filePos;
+                $startAttributes['startFilePos'] = $token->pos;
             }
 
-            if (\is_string($token)) {
+            $id = $token->id;
-                $value = $token;
+            if (isset($this->dropTokens[$id])) {
-                if (isset($token[1])) {
+                if (\T_COMMENT === $id || \T_DOC_COMMENT === $id) {
-                    // bug in token_get_all
+                    if ($this->attributeCommentsUsed) {
-                    $this->filePos += 2;
+                        $comment = \T_DOC_COMMENT === $id
-                    $id = ord('"');
+                            ? new Comment\Doc($token->text, $token->line, $token->pos, $this->pos,
-                } else {
+                                $token->getEndLine(), $token->getEndPos() - 1, $this->pos)
-                    $this->filePos += 1;
+                            : new Comment($token->text, $token->line, $token->pos, $this->pos,
-                    $id = ord($token);
+                                $token->getEndLine(), $token->getEndPos() - 1, $this->pos);
-                }
-            } elseif (!isset($this->dropTokens[$token[0]])) {
-                $value = $token[1];
-                $id = $this->tokenMap[$token[0]];
-                if (\T_CLOSE_TAG === $token[0]) {
-                    $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
-                } elseif (\T_INLINE_HTML === $token[0]) {
-                    $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
-                }
-
-                $this->line += substr_count($value, "\n");
-                $this->filePos += \strlen($value);
-            } else {
-                if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
-                    if (isset($this->usedAttributes['comments'])) {
-                        $comment = \T_DOC_COMMENT === $token[0]
-                            ? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
-                            : new Comment($token[1], $this->line, $this->filePos, $this->pos);
                         $startAttributes['comments'][] = $comment;
                     }
                 }
-
-                $this->line += substr_count($token[1], "\n");
-                $this->filePos += \strlen($token[1]);
                 continue;
             }
 
-            if (isset($this->usedAttributes['endLine'])) {
+            $value = $token->text;
-                $endAttributes['endLine'] = $this->line;
+            if (\T_CLOSE_TAG === $token->id) {
+                $this->prevCloseTagHasNewline = false !== strpos($value, "\n")
+                    || false !== strpos($value, "\r");
+            } elseif (\T_INLINE_HTML === $token->id) {
+                $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
             }
-            if (isset($this->usedAttributes['endTokenPos'])) {
+
+            // Fetch the end line/pos from the next token (if available) instead of recomputing it.
+            $nextToken = $this->tokens[$this->pos + 1] ?? null;
+            if ($this->attributeEndLineUsed) {
+                $endAttributes['endLine'] = $nextToken ? $nextToken->line : $token->getEndLine();
+            }
+            if ($this->attributeEndTokenPosUsed) {
                 $endAttributes['endTokenPos'] = $this->pos;
             }
-            if (isset($this->usedAttributes['endFilePos'])) {
+            if ($this->attributeEndFilePosUsed) {
-                $endAttributes['endFilePos'] = $this->filePos - 1;
+                $endAttributes['endFilePos'] = ($nextToken ? $nextToken->pos : $token->getEndPos()) - 1;
             }
 
             return $id;
         }
-
-        throw new \RuntimeException('Reached end of lexer loop');
     }
 
     /**
      * Returns the token array for current code.
      *
-     * The token array is in the same format as provided by the
+     * The token array is in the same format as provided by the PhpToken::tokenize() method in
-     * token_get_all() function and does not discard tokens (i.e.
+     * PHP 8.0. The tokens are instances of PhpParser\Token, to abstract over a polyfill
-     * whitespace and comments are included). The token position
+     * implementation in earlier PHP version.
-     * attributes are against this token array.
      *
-     * @return array Array of tokens in token_get_all() format
+     * The token array is terminated by a sentinel token with token ID 0.
+     * The token array does not discard any tokens (i.e. whitespace and comments are included).
+     * The token position attributes are against this token array.
+     *
+     * @return Token[] Array of tokens
      */
-    public function getTokens() : array {
+    public function getTokens(): array {
         return $this->tokens;
     }
 
@@ -311,68 +259,12 @@ class Lexer
      *
      * @return string Remaining text
      */
-    public function handleHaltCompiler() : string {
+    public function handleHaltCompiler(): string {
-        // text after T_HALT_COMPILER, still including ();
+        // Prevent the lexer from returning any further tokens.
-        $textAfter = substr($this->code, $this->filePos);
+        $nextToken = $this->tokens[$this->pos + 1];
+        $this->pos = \count($this->tokens) - 2;
 
-        // ensure that it is followed by ();
+        // Return text after __halt_compiler.
-        // this simplifies the situation, by not allowing any comments
+        return $nextToken->id === \T_INLINE_HTML ? $nextToken->text : '';
-        // in between of the tokens.
-        if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
-            throw new Error('__HALT_COMPILER must be followed by "();"');
-        }
-
-        // prevent the lexer from returning any further tokens
-        $this->pos = count($this->tokens);
-
-        // return with (); removed
-        return substr($textAfter, strlen($matches[0]));
-    }
-
-    /**
-     * Creates the token map.
-     *
-     * The token map maps the PHP internal token identifiers
-     * to the identifiers used by the Parser. Additionally it
-     * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
-     *
-     * @return array The token map
-     */
-    protected function createTokenMap() : array {
-        $tokenMap = [];
-
-        // 256 is the minimum possible token number, as everything below
-        // it is an ASCII value
-        for ($i = 256; $i < 1000; ++$i) {
-            if (\T_DOUBLE_COLON === $i) {
-                // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
-                $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
-            } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
-                // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
-                $tokenMap[$i] = Tokens::T_ECHO;
-            } elseif(\T_CLOSE_TAG === $i) {
-                // T_CLOSE_TAG is equivalent to ';'
-                $tokenMap[$i] = ord(';');
-            } elseif ('UNKNOWN' !== $name = token_name($i)) {
-                if ('T_HASHBANG' === $name) {
-                    // HHVM uses a special token for #! hashbang lines
-                    $tokenMap[$i] = Tokens::T_INLINE_HTML;
-                } elseif (defined($name = Tokens::class . '::' . $name)) {
-                    // Other tokens can be mapped directly
-                    $tokenMap[$i] = constant($name);
-                }
-            }
-        }
-
-        // HHVM uses a special token for numbers that overflow to double
-        if (defined('T_ONUMBER')) {
-            $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
-        }
-        // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
-        if (defined('T_COMPILER_HALT_OFFSET')) {
-            $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
-        }
-
-        return $tokenMap;
     }
 }

lib/PhpParser/Lexer/Emulative.php

@@ -4,27 +4,93 @@ namespace PhpParser\Lexer;
 
 use PhpParser\Error;
 use PhpParser\ErrorHandler;
+use PhpParser\Lexer;
+use PhpParser\Lexer\TokenEmulator\AttributeEmulator;
+use PhpParser\Lexer\TokenEmulator\EnumTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\ExplicitOctalEmulator;
+use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator;
+use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\MatchTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NullsafeTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
+use PhpParser\Lexer\TokenEmulator\ReadonlyTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\ReverseEmulator;
+use PhpParser\Lexer\TokenEmulator\TokenEmulator;
+use PhpParser\PhpVersion;
 
-class Emulative extends \PhpParser\Lexer
+class Emulative extends Lexer {
-{
+    /** @var mixed[] Patches used to reverse changes introduced in the code */
-    const PHP_7_3 = '7.3.0dev';
+    private $patches = [];
+
+    /** @var TokenEmulator[] */
+    private $emulators = [];
+
+    /** @var PhpVersion */
+    private $targetPhpVersion;
+    /** @var PhpVersion */
+    private $hostPhpVersion;
 
     /**
-     * @var array Patches used to reverse changes introduced in the code
+     * @param mixed[] $options Lexer options. In addition to the usual options, accepts a
+     *                         'phpVersion' (PhpVersion object or string) that specifies the
+     *                         version to emulate. Defaults to newest supported.
      */
-    private $patches;
+    public function __construct(array $options = []) {
+        $version = $options['phpVersion'] ?? PhpVersion::getNewestSupported();
+        if (!$version instanceof PhpVersion) {
+            $version = PhpVersion::fromString($version);
+        }
+        $this->targetPhpVersion = $version;
+        $this->hostPhpVersion = PhpVersion::getHostVersion();
+        unset($options['phpVersion']);
 
-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+        parent::__construct($options);
-        $this->patches = [];
+
-        $preparedCode = $this->prepareCode($code);
+        $emulators = [
-        if (null === $preparedCode) {
+            new FlexibleDocStringEmulator(),
+            new FnTokenEmulator(),
+            new MatchTokenEmulator(),
+            new CoaleseEqualTokenEmulator(),
+            new NumericLiteralSeparatorEmulator(),
+            new NullsafeTokenEmulator(),
+            new AttributeEmulator(),
+            new EnumTokenEmulator(),
+            new ReadonlyTokenEmulator(),
+            new ExplicitOctalEmulator(),
+        ];
+
+        // Collect emulators that are relevant for the PHP version we're running
+        // and the PHP version we're targeting for emulation.
+        foreach ($emulators as $emulator) {
+            $emulatorPhpVersion = $emulator->getPhpVersion();
+            if ($this->isForwardEmulationNeeded($emulatorPhpVersion)) {
+                $this->emulators[] = $emulator;
+            } elseif ($this->isReverseEmulationNeeded($emulatorPhpVersion)) {
+                $this->emulators[] = new ReverseEmulator($emulator);
+            }
+        }
+    }
+
+    public function startLexing(string $code, ?ErrorHandler $errorHandler = null) {
+        $emulators = array_filter($this->emulators, function ($emulator) use ($code) {
+            return $emulator->isEmulationNeeded($code);
+        });
+
+        if (empty($emulators)) {
             // Nothing to emulate, yay
             parent::startLexing($code, $errorHandler);
             return;
         }
 
+        $this->patches = [];
+        foreach ($emulators as $emulator) {
+            $code = $emulator->preprocessCode($code, $this->patches);
+        }
+
         $collector = new ErrorHandler\Collecting();
-        parent::startLexing($preparedCode, $collector);
+        parent::startLexing($code, $collector);
+        $this->sortPatches();
         $this->fixupTokens();
 
         $errors = $collector->getErrors();
@@ -34,92 +100,47 @@ class Emulative extends \PhpParser\Lexer
                 $errorHandler->handleError($error);
             }
         }
+
+        foreach ($emulators as $emulator) {
+            $this->tokens = $emulator->emulate($code, $this->tokens);
+        }
     }
 
-    /**
+    private function isForwardEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
-     * Prepares code for emulation. If nothing has to be emulated null is returned.
+        return $this->hostPhpVersion->older($emulatorPhpVersion)
-     *
+            && $this->targetPhpVersion->newerOrEqual($emulatorPhpVersion);
-     * @param string $code
+    }
-     * @return null|string
-     */
-    private function prepareCode(string $code) {
-        if (version_compare(\PHP_VERSION, self::PHP_7_3, '>=')) {
-            return null;
-        }
 
-        if (strpos($code, '<<<') === false) {
+    private function isReverseEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
-            // Definitely doesn't contain heredoc/nowdoc
+        return $this->hostPhpVersion->newerOrEqual($emulatorPhpVersion)
-            return null;
+            && $this->targetPhpVersion->older($emulatorPhpVersion);
-        }
+    }
 
-        $flexibleDocStringRegex = <<<'REGEX'
+    private function sortPatches() {
-/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
+        // Patches may be contributed by different emulators.
-(?:.*\r?\n)*?
+        // Make sure they are sorted by increasing patch position.
-(?<indentation>\h*)\2(?![a-zA-Z_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
+        usort($this->patches, function ($p1, $p2) {
-REGEX;
+            return $p1[0] <=> $p2[0];
-        if (!preg_match_all($flexibleDocStringRegex, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
+        });
-            // No heredoc/nowdoc found
-            return null;
-        }
-
-        // Keep track of how much we need to adjust string offsets due to the modifications we
-        // already made
-        $posDelta = 0;
-        foreach ($matches as $match) {
-            $indentation = $match['indentation'][0];
-            $indentationStart = $match['indentation'][1];
-
-            $separator = $match['separator'][0];
-            $separatorStart = $match['separator'][1];
-
-            if ($indentation === '' && $separator !== '') {
-                // Ordinary heredoc/nowdoc
-                continue;
-            }
-
-            if ($indentation !== '') {
-                // Remove indentation
-                $indentationLen = strlen($indentation);
-                $code = substr_replace($code, '', $indentationStart + $posDelta, $indentationLen);
-                $this->patches[] = [$indentationStart + $posDelta, 'add', $indentation];
-                $posDelta -= $indentationLen;
-            }
-
-            if ($separator === '') {
-                // Insert newline as separator
-                $code = substr_replace($code, "\n", $separatorStart + $posDelta, 0);
-                $this->patches[] = [$separatorStart + $posDelta, 'remove', "\n"];
-                $posDelta += 1;
-            }
-        }
-
-        if (empty($this->patches)) {
-            // We did not end up emulating anything
-            return null;
-        }
-
-        return $code;
     }
 
     private function fixupTokens() {
-        assert(count($this->patches) > 0);
+        if (\count($this->patches) === 0) {
+            return;
+        }
 
         // Load first patch
         $patchIdx = 0;
         list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
 
         // We use a manual loop over the tokens, because we modify the array on the fly
-        $pos = 0;
+        $posDelta = 0;
         for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
             $token = $this->tokens[$i];
-            if (\is_string($token)) {
+            $pos = $token->pos;
-                // We assume that patches don't apply to string tokens
+            $token->pos += $posDelta;
-                $pos += \strlen($token);
+            $localPosDelta = 0;
-                continue;
+            $len = \strlen($token->text);
-            }
-
-            $len = \strlen($token[1]);
-            $posDelta = 0;
             while ($patchPos >= $pos && $patchPos < $pos + $len) {
                 $patchTextLen = \strlen($patchText);
                 if ($patchType === 'remove') {
@@ -130,17 +151,22 @@ REGEX;
                         $c--;
                     } else {
                         // Remove from token string
-                        $this->tokens[$i][1] = substr_replace(
+                        $token->text = substr_replace(
-                            $token[1], '', $patchPos - $pos + $posDelta, $patchTextLen
+                            $token->text, '', $patchPos - $pos + $localPosDelta, $patchTextLen
                         );
-                        $posDelta -= $patchTextLen;
+                        $localPosDelta -= $patchTextLen;
                     }
                 } elseif ($patchType === 'add') {
                     // Insert into the token string
-                    $this->tokens[$i][1] = substr_replace(
+                    $token->text = substr_replace(
-                        $token[1], $patchText, $patchPos - $pos + $posDelta, 0
+                        $token->text, $patchText, $patchPos - $pos + $localPosDelta, 0
+                    );
+                    $localPosDelta += $patchTextLen;
+                } elseif ($patchType === 'replace') {
+                    // Replace inside the token string
+                    $token->text = substr_replace(
+                        $token->text, $patchText, $patchPos - $pos + $localPosDelta, $patchTextLen
                     );
-                    $posDelta += $patchTextLen;
                 } else {
                     assert(false);
                 }
@@ -148,22 +174,16 @@ REGEX;
                 // Fetch the next patch
                 $patchIdx++;
                 if ($patchIdx >= \count($this->patches)) {
-                    // No more patches, we're done
+                    // No more patches. However, we still need to adjust position.
-                    return;
+                    $patchPos = \PHP_INT_MAX;
+                    break;
                 }
 
                 list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
-
-                // Multiple patches may apply to the same token. Reload the current one to check
-                // If the new patch applies
-                $token = $this->tokens[$i];
             }
 
-            $pos += $len;
+            $posDelta += $localPosDelta;
         }
-
-        // A patch did not apply
-        assert(false);
     }
 
     /**
@@ -187,7 +207,7 @@ REGEX;
                 if ($patchType === 'add') {
                     $posDelta += strlen($patchText);
                     $lineDelta += substr_count($patchText, "\n");
-                } else {
+                } elseif ($patchType === 'remove') {
                     $posDelta -= strlen($patchText);
                     $lineDelta -= substr_count($patchText, "\n");
                 }

lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php

@@ -0,0 +1,49 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+final class AttributeEmulator extends TokenEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 0);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '#[') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way.
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            if ($token->text === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1]->text === '[') {
+                array_splice($tokens, $i, 2, [
+                    new Token(\T_ATTRIBUTE, '#[', $token->line, $token->pos),
+                ]);
+                $c--;
+                continue;
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // TODO
+        return $tokens;
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        $pos = 0;
+        while (false !== $pos = strpos($code, '#[', $pos)) {
+            // Replace #[ with %[
+            $code[$pos] = '%';
+            $patches[] = [$pos, 'replace', '#'];
+            $pos += 2;
+        }
+        return $code;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php

@@ -0,0 +1,40 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+final class CoaleseEqualTokenEmulator extends TokenEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(7, 4);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '??=') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            if (isset($tokens[$i + 1])) {
+                if ($token->id === T_COALESCE && $tokens[$i + 1]->text === '=') {
+                    array_splice($tokens, $i, 2, [
+                        new Token(\T_COALESCE_EQUAL, '??=', $token->line, $token->pos),
+                    ]);
+                    $c--;
+                    continue;
+                }
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // ??= was not valid code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/EnumTokenEmulator.php

@@ -0,0 +1,26 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+final class EnumTokenEmulator extends KeywordEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 1);
+    }
+
+    public function getKeywordString(): string {
+        return 'enum';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_ENUM;
+    }
+
+    protected function isKeywordContext(array $tokens, int $pos): bool {
+        return parent::isKeywordContext($tokens, $pos)
+            && isset($tokens[$pos + 2])
+            && $tokens[$pos + 1]->id === \T_WHITESPACE
+            && $tokens[$pos + 2]->id === \T_STRING;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/ExplicitOctalEmulator.php

@@ -0,0 +1,45 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+class ExplicitOctalEmulator extends TokenEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 1);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '0o') !== false || strpos($code, '0O') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            if ($token->id == \T_LNUMBER && $token->text === '0' &&
+                isset($tokens[$i + 1]) && $tokens[$i + 1]->id == \T_STRING &&
+                preg_match('/[oO][0-7]+(?:_[0-7]+)*/', $tokens[$i + 1]->text)
+            ) {
+                $tokenKind = $this->resolveIntegerOrFloatToken($tokens[$i + 1]->text);
+                array_splice($tokens, $i, 2, [
+                    new Token($tokenKind, '0' . $tokens[$i + 1]->text, $token->line, $token->pos),
+                ]);
+                $c--;
+            }
+        }
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int {
+        $str = substr($str, 1);
+        $str = str_replace('_', '', $str);
+        $num = octdec($str);
+        return is_float($num) ? \T_DNUMBER : \T_LNUMBER;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // Explicit octals were not legal code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/FlexibleDocStringEmulator.php

@@ -0,0 +1,71 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+final class FlexibleDocStringEmulator extends TokenEmulator {
+    private const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
+/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
+(?:.*\r?\n)*?
+(?<indentation>\h*)\2(?![a-zA-Z0-9_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
+REGEX;
+
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(7, 3);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '<<<') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        // Handled by preprocessing + fixup.
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // Not supported.
+        return $tokens;
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
+            // No heredoc/nowdoc found
+            return $code;
+        }
+
+        // Keep track of how much we need to adjust string offsets due to the modifications we
+        // already made
+        $posDelta = 0;
+        foreach ($matches as $match) {
+            $indentation = $match['indentation'][0];
+            $indentationStart = $match['indentation'][1];
+
+            $separator = $match['separator'][0];
+            $separatorStart = $match['separator'][1];
+
+            if ($indentation === '' && $separator !== '') {
+                // Ordinary heredoc/nowdoc
+                continue;
+            }
+
+            if ($indentation !== '') {
+                // Remove indentation
+                $indentationLen = strlen($indentation);
+                $code = substr_replace($code, '', $indentationStart + $posDelta, $indentationLen);
+                $patches[] = [$indentationStart + $posDelta, 'add', $indentation];
+                $posDelta -= $indentationLen;
+            }
+
+            if ($separator === '') {
+                // Insert newline as separator
+                $code = substr_replace($code, "\n", $separatorStart + $posDelta, 0);
+                $patches[] = [$separatorStart + $posDelta, 'remove', "\n"];
+                $posDelta += 1;
+            }
+        }
+
+        return $code;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php

@@ -0,0 +1,19 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+final class FnTokenEmulator extends KeywordEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(7, 4);
+    }
+
+    public function getKeywordString(): string {
+        return 'fn';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_FN;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/KeywordEmulator.php

@@ -0,0 +1,56 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Token;
+
+abstract class KeywordEmulator extends TokenEmulator {
+    abstract public function getKeywordString(): string;
+    abstract public function getKeywordToken(): int;
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos(strtolower($code), $this->getKeywordString()) !== false;
+    }
+
+    /** @param Token[] $tokens */
+    protected function isKeywordContext(array $tokens, int $pos): bool {
+        $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $pos);
+        return $previousNonSpaceToken === null || $previousNonSpaceToken->id !== \T_OBJECT_OPERATOR;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        $keywordString = $this->getKeywordString();
+        foreach ($tokens as $i => $token) {
+            if ($token->id === T_STRING && strtolower($token->text) === $keywordString
+                    && $this->isKeywordContext($tokens, $i)) {
+                $token->id = $this->getKeywordToken();
+            }
+        }
+
+        return $tokens;
+    }
+
+    /** @param Token[] $tokens */
+    private function getPreviousNonSpaceToken(array $tokens, int $start): ?Token {
+        for ($i = $start - 1; $i >= 0; --$i) {
+            if ($tokens[$i]->id === T_WHITESPACE) {
+                continue;
+            }
+
+            return $tokens[$i];
+        }
+
+        return null;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        $keywordToken = $this->getKeywordToken();
+        foreach ($tokens as $i => $token) {
+            if ($token->id === $keywordToken) {
+                $token->id = \T_STRING;
+            }
+        }
+
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/MatchTokenEmulator.php

@@ -0,0 +1,19 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+final class MatchTokenEmulator extends KeywordEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 0);
+    }
+
+    public function getKeywordString(): string {
+        return 'match';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_MATCH;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/NullsafeTokenEmulator.php

@@ -0,0 +1,60 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+final class NullsafeTokenEmulator extends TokenEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 0);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '?->') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            if ($token->text === '?' && isset($tokens[$i + 1]) && $tokens[$i + 1]->id === \T_OBJECT_OPERATOR) {
+                array_splice($tokens, $i, 2, [
+                    new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', $token->line, $token->pos),
+                ]);
+                $c--;
+                continue;
+            }
+
+            // Handle ?-> inside encapsed string.
+            if ($token->id === \T_ENCAPSED_AND_WHITESPACE && isset($tokens[$i - 1])
+                && $tokens[$i - 1]->id === \T_VARIABLE
+                && preg_match('/^\?->([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)/', $token->text, $matches)
+            ) {
+                $replacement = [
+                    new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', $token->line, $token->pos),
+                    new Token(\T_STRING, $matches[1], $token->line, $token->pos + 3),
+                ];
+                $matchLen = \strlen($matches[0]);
+                if ($matchLen !== \strlen($token->text)) {
+                    $replacement[] = new Token(
+                        \T_ENCAPSED_AND_WHITESPACE,
+                        \substr($token->text, $matchLen),
+                        $token->line, $token->pos + $matchLen
+                    );
+                }
+                array_splice($tokens, $i, 1, $replacement);
+                $c += \count($replacement) - 1;
+                continue;
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // ?-> was not valid code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -0,0 +1,95 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+final class NumericLiteralSeparatorEmulator extends TokenEmulator {
+    private const BIN = '(?:0b[01]+(?:_[01]+)*)';
+    private const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
+    private const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
+    private const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
+    private const EXP = '(?:e[+-]?' . self::DEC . ')';
+    private const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
+    private const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
+
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(7, 4);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return preg_match('~[0-9]_[0-9]~', $code)
+            || preg_match('~0x[0-9a-f]+_[0-9a-f]~i', $code);
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            $tokenLen = \strlen($token->text);
+
+            if ($token->id !== \T_LNUMBER && $token->id !== \T_DNUMBER) {
+                continue;
+            }
+
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $token->pos);
+            assert($res, "No number at number token position");
+
+            $match = $matches[0];
+            $matchLen = \strlen($match);
+            if ($matchLen === $tokenLen) {
+                // Original token already holds the full number.
+                continue;
+            }
+
+            $tokenKind = $this->resolveIntegerOrFloatToken($match);
+            $newTokens = [new Token($tokenKind, $match, $token->line, $token->pos)];
+
+            $numTokens = 1;
+            $len = $tokenLen;
+            while ($matchLen > $len) {
+                $nextToken = $tokens[$i + $numTokens];
+                $nextTokenText = $nextToken->text;
+                $nextTokenLen = \strlen($nextTokenText);
+
+                $numTokens++;
+                if ($matchLen < $len + $nextTokenLen) {
+                    // Split trailing characters into a partial token.
+                    $partialText = substr($nextTokenText, $matchLen - $len);
+                    $newTokens[] = new Token($nextToken->id, $partialText, $nextToken->line, $nextToken->pos);
+                    break;
+                }
+
+                $len += $nextTokenLen;
+            }
+
+            array_splice($tokens, $i, $numTokens, $newTokens);
+            $c -= $numTokens - \count($newTokens);
+        }
+
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int {
+        $str = str_replace('_', '', $str);
+
+        if (stripos($str, '0b') === 0) {
+            $num = bindec($str);
+        } elseif (stripos($str, '0x') === 0) {
+            $num = hexdec($str);
+        } elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
+            $num = octdec($str);
+        } else {
+            $num = +$str;
+        }
+
+        return is_float($num) ? T_DNUMBER : T_LNUMBER;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // Numeric separators were not legal code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/ReadonlyTokenEmulator.php

@@ -0,0 +1,31 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+final class ReadonlyTokenEmulator extends KeywordEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 1);
+    }
+
+    public function getKeywordString(): string {
+        return 'readonly';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_READONLY;
+    }
+
+    protected function isKeywordContext(array $tokens, int $pos): bool {
+        if (!parent::isKeywordContext($tokens, $pos)) {
+            return false;
+        }
+        // Support "function readonly("
+        return !(isset($tokens[$pos + 1]) &&
+                 ($tokens[$pos + 1]->text === '(' ||
+                  ($tokens[$pos + 1]->id === \T_WHITESPACE &&
+                   isset($tokens[$pos + 2]) &&
+                   $tokens[$pos + 2]->text === '(')));
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/ReverseEmulator.php

@@ -0,0 +1,37 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+/**
+ * Reverses emulation direction of the inner emulator.
+ */
+final class ReverseEmulator extends TokenEmulator {
+    /** @var TokenEmulator Inner emulator */
+    private $emulator;
+
+    public function __construct(TokenEmulator $emulator) {
+        $this->emulator = $emulator;
+    }
+
+    public function getPhpVersion(): PhpVersion {
+        return $this->emulator->getPhpVersion();
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return $this->emulator->isEmulationNeeded($code);
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        return $this->emulator->reverseEmulate($code, $tokens);
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        return $this->emulator->emulate($code, $tokens);
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        return $code;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/TokenEmulator.php

@@ -0,0 +1,28 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+/** @internal */
+abstract class TokenEmulator {
+    abstract public function getPhpVersion(): PhpVersion;
+
+    abstract public function isEmulationNeeded(string $code): bool;
+
+    /**
+     * @param Token[] Original tokens
+     * @return Token[] Modified Tokens
+     */
+    abstract public function emulate(string $code, array $tokens): array;
+
+    /**
+     * @return Token[] Modified Tokens
+     */
+    abstract public function reverseEmulate(string $code, array $tokens): array;
+
+    public function preprocessCode(string $code, array &$patches): string {
+        return $code;
+    }
+}

lib/PhpParser/Modifiers.php

@@ -0,0 +1,19 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser;
+
+/**
+ * Modifiers used (as a bit mask) by various flags subnodes, for example on classes, functions,
+ * properties and constants.
+ */
+final class Modifiers {
+    public const PUBLIC    =  1;
+    public const PROTECTED =  2;
+    public const PRIVATE   =  4;
+    public const STATIC    =  8;
+    public const ABSTRACT  = 16;
+    public const FINAL     = 32;
+    public const READONLY  = 64;
+
+    public const VISIBILITY_MASK = 1 | 2 | 4;
+}

lib/PhpParser/NameContext.php

@@ -6,8 +6,7 @@ use PhpParser\Node\Name;
 use PhpParser\Node\Name\FullyQualified;
 use PhpParser\Node\Stmt;
 
-class NameContext
+class NameContext {
-{
     /** @var null|Name Current namespace */
     protected $namespace;
 
@@ -36,7 +35,7 @@ class NameContext
      *
      * @param Name|null $namespace Null is the global namespace
      */
-    public function startNamespace(Name $namespace = null) {
+    public function startNamespace(?Name $namespace = null) {
         $this->namespace = $namespace;
         $this->origAliases = $this->aliases = [
             Stmt\Use_::TYPE_NORMAL   => [],
@@ -87,7 +86,7 @@ class NameContext
      *
      * @return null|Name Namespace (or null if global namespace)
      */
-    public function getNamespace() {
+    public function getNamespace(): ?Name {
         return $this->namespace;
     }
 
@@ -99,7 +98,7 @@ class NameContext
      *
      * @return null|Name Resolved name, or null if static resolution is not possible
      */
-    public function getResolvedName(Name $name, int $type) {
+    public function getResolvedName(Name $name, int $type): ?Name {
         // don't resolve special class names
         if ($type === Stmt\Use_::TYPE_NORMAL && $name->isSpecialClassName()) {
             if (!$name->isUnqualified()) {
@@ -142,7 +141,7 @@ class NameContext
      *
      * @return Name Resolved name
      */
-    public function getResolvedClassName(Name $name) : Name {
+    public function getResolvedClassName(Name $name): Name {
         return $this->getResolvedName($name, Stmt\Use_::TYPE_NORMAL);
     }
 
@@ -154,7 +153,7 @@ class NameContext
      *
      * @return Name[] Possible representations of the name
      */
-    public function getPossibleNames(string $name, int $type) : array {
+    public function getPossibleNames(string $name, int $type): array {
         $lcName = strtolower($name);
 
         if ($type === Stmt\Use_::TYPE_NORMAL) {
@@ -210,7 +209,7 @@ class NameContext
      *
      * @return Name Shortest representation
      */
-    public function getShortName(string $name, int $type) : Name {
+    public function getShortName(string $name, int $type): Name {
         $possibleNames = $this->getPossibleNames($name, $type);
 
         // Find shortest name
@@ -224,7 +223,7 @@ class NameContext
             }
         }
 
-       return $shortestName;
+        return $shortestName;
     }
 
     private function resolveAlias(Name $name, $type) {

lib/PhpParser/Node.php

@@ -2,28 +2,27 @@
 
 namespace PhpParser;
 
-interface Node
+interface Node {
-{
     /**
      * Gets the type of the node.
      *
      * @return string Type of the node
      */
-    public function getType() : string;
+    public function getType(): string;
 
     /**
      * Gets the names of the sub nodes.
      *
      * @return array Names of sub nodes
      */
-    public function getSubNodeNames() : array;
+    public function getSubNodeNames(): array;
 
     /**
      * Gets line the node started in (alias of getStartLine).
      *
      * @return int Start line (or -1 if not available)
      */
-    public function getLine() : int;
+    public function getLine(): int;
 
     /**
      * Gets line the node started in.
@@ -32,7 +31,7 @@ interface Node
      *
      * @return int Start line (or -1 if not available)
      */
-    public function getStartLine() : int;
+    public function getStartLine(): int;
 
     /**
      * Gets the line the node ended in.
@@ -41,7 +40,7 @@ interface Node
      *
      * @return int End line (or -1 if not available)
      */
-    public function getEndLine() : int;
+    public function getEndLine(): int;
 
     /**
      * Gets the token offset of the first token that is part of this node.
@@ -52,7 +51,7 @@ interface Node
      *
      * @return int Token start position (or -1 if not available)
      */
-    public function getStartTokenPos() : int;
+    public function getStartTokenPos(): int;
 
     /**
      * Gets the token offset of the last token that is part of this node.
@@ -63,7 +62,7 @@ interface Node
      *
      * @return int Token end position (or -1 if not available)
      */
-    public function getEndTokenPos() : int;
+    public function getEndTokenPos(): int;
 
     /**
      * Gets the file offset of the first character that is part of this node.
@@ -72,7 +71,7 @@ interface Node
      *
      * @return int File start position (or -1 if not available)
      */
-    public function getStartFilePos() : int;
+    public function getStartFilePos(): int;
 
     /**
      * Gets the file offset of the last character that is part of this node.
@@ -81,7 +80,7 @@ interface Node
      *
      * @return int File end position (or -1 if not available)
      */
-    public function getEndFilePos() : int;
+    public function getEndFilePos(): int;
 
     /**
      * Gets all comments directly preceding this node.
@@ -90,16 +89,14 @@ interface Node
      *
      * @return Comment[]
      */
-    public function getComments() : array;
+    public function getComments(): array;
 
     /**
      * Gets the doc comment of the node.
      *
-     * The doc comment has to be the last comment associated with the node.
-     *
      * @return null|Comment\Doc Doc comment object or null
      */
-    public function getDocComment();
+    public function getDocComment(): ?Comment\Doc;
 
     /**
      * Sets the doc comment of the node.
@@ -125,7 +122,7 @@ interface Node
      *
      * @return bool
      */
-    public function hasAttribute(string $key) : bool;
+    public function hasAttribute(string $key): bool;
 
     /**
      * Returns the value of an attribute.
@@ -142,7 +139,7 @@ interface Node
      *
      * @return array
      */
-    public function getAttributes() : array;
+    public function getAttributes(): array;
 
     /**
      * Replaces all the attributes of this node.

lib/PhpParser/Node/Arg.php

@@ -4,8 +4,9 @@ namespace PhpParser\Node;
 
 use PhpParser\NodeAbstract;
 
-class Arg extends NodeAbstract
+class Arg extends NodeAbstract {
-{
+    /** @var Identifier|null Parameter name (for named parameters) */
+    public $name;
     /** @var Expr Value to pass */
     public $value;
     /** @var bool Whether to pass by ref */
@@ -20,19 +21,24 @@ class Arg extends NodeAbstract
      * @param bool  $byRef      Whether to pass by ref
      * @param bool  $unpack     Whether to unpack the argument
      * @param array $attributes Additional attributes
+     * @param Identifier|null $name Parameter name (for named parameters)
      */
-    public function __construct(Expr $value, bool $byRef = false, bool $unpack = false, array $attributes = []) {
+    public function __construct(
-        parent::__construct($attributes);
+        Expr $value, bool $byRef = false, bool $unpack = false, array $attributes = [],
+        ?Identifier $name = null
+    ) {
+        $this->attributes = $attributes;
+        $this->name = $name;
         $this->value = $value;
         $this->byRef = $byRef;
         $this->unpack = $unpack;
     }
 
-    public function getSubNodeNames() : array {
+    public function getSubNodeNames(): array {
-        return ['value', 'byRef', 'unpack'];
+        return ['name', 'value', 'byRef', 'unpack'];
     }
 
-    public function getType() : string {
+    public function getType(): string {
         return 'Arg';
     }
 }

lib/PhpParser/Node/ArrayItem.php

@@ -0,0 +1,43 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\NodeAbstract;
+
+class ArrayItem extends NodeAbstract {
+    /** @var null|Expr Key */
+    public $key;
+    /** @var Expr Value */
+    public $value;
+    /** @var bool Whether to assign by reference */
+    public $byRef;
+    /** @var bool Whether to unpack the argument */
+    public $unpack;
+
+    /**
+     * Constructs an array item node.
+     *
+     * @param Expr      $value      Value
+     * @param null|Expr $key        Key
+     * @param bool      $byRef      Whether to assign by reference
+     * @param array     $attributes Additional attributes
+     */
+    public function __construct(Expr $value, ?Expr $key = null, bool $byRef = false, array $attributes = [], bool $unpack = false) {
+        $this->attributes = $attributes;
+        $this->key = $key;
+        $this->value = $value;
+        $this->byRef = $byRef;
+        $this->unpack = $unpack;
+    }
+
+    public function getSubNodeNames(): array {
+        return ['key', 'value', 'byRef', 'unpack'];
+    }
+
+    public function getType(): string {
+        return 'ArrayItem';
+    }
+}
+
+// @deprecated compatibility alias
+class_alias(ArrayItem::class, Expr\ArrayItem::class);

lib/PhpParser/Node/Attribute.php

@@ -0,0 +1,33 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\Node;
+use PhpParser\NodeAbstract;
+
+class Attribute extends NodeAbstract {
+    /** @var Name Attribute name */
+    public $name;
+
+    /** @var Arg[] Attribute arguments */
+    public $args;
+
+    /**
+     * @param Node\Name $name       Attribute name
+     * @param Arg[]     $args       Attribute arguments
+     * @param array     $attributes Additional node attributes
+     */
+    public function __construct(Name $name, array $args = [], array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->name = $name;
+        $this->args = $args;
+    }
+
+    public function getSubNodeNames(): array {
+        return ['name', 'args'];
+    }
+
+    public function getType(): string {
+        return 'Attribute';
+    }
+}

lib/PhpParser/Node/AttributeGroup.php

@@ -0,0 +1,27 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\NodeAbstract;
+
+class AttributeGroup extends NodeAbstract {
+    /** @var Attribute[] Attributes */
+    public $attrs;
+
+    /**
+     * @param Attribute[] $attrs PHP attributes
+     * @param array $attributes Additional node attributes
+     */
+    public function __construct(array $attrs, array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->attrs = $attrs;
+    }
+
+    public function getSubNodeNames(): array {
+        return ['attrs'];
+    }
+
+    public function getType(): string {
+        return 'AttributeGroup';
+    }
+}

lib/PhpParser/Node/ClosureUse.php

@@ -0,0 +1,36 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\NodeAbstract;
+
+class ClosureUse extends NodeAbstract {
+    /** @var Expr\Variable Variable to use */
+    public $var;
+    /** @var bool Whether to use by reference */
+    public $byRef;
+
+    /**
+     * Constructs a closure use node.
+     *
+     * @param Expr\Variable $var        Variable to use
+     * @param bool          $byRef      Whether to use by reference
+     * @param array         $attributes Additional attributes
+     */
+    public function __construct(Expr\Variable $var, bool $byRef = false, array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->var = $var;
+        $this->byRef = $byRef;
+    }
+
+    public function getSubNodeNames(): array {
+        return ['var', 'byRef'];
+    }
+
+    public function getType(): string {
+        return 'ClosureUse';
+    }
+}
+
+// @deprecated compatibility alias
+class_alias(ClosureUse::class, Expr\ClosureUse::class);

lib/PhpParser/Node/ComplexType.php

@@ -0,0 +1,13 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\NodeAbstract;
+
+/**
+ * This is a base class for complex types, including nullable types and union types.
+ *
+ * It does not provide any shared behavior and exists only for type-checking purposes.
+ */
+abstract class ComplexType extends NodeAbstract {
+}

lib/PhpParser/Node/Const_.php

@@ -4,16 +4,15 @@ namespace PhpParser\Node;
 
 use PhpParser\NodeAbstract;
 
-/**
+class Const_ extends NodeAbstract {
- * @property Name $namespacedName Namespaced name (for class constants, if using NameResolver)
- */
-class Const_ extends NodeAbstract
-{
     /** @var Identifier Name */
     public $name;
     /** @var Expr Value */
     public $value;
 
+    /** @var Name|null Namespaced name (if using NameResolver) */
+    public $namespacedName;
+
     /**
      * Constructs a const node for use in class const and const statements.
      *
@@ -22,16 +21,16 @@ class Const_ extends NodeAbstract
      * @param array             $attributes Additional attributes
      */
     public function __construct($name, Expr $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
         $this->value = $value;
     }
 
-    public function getSubNodeNames() : array {
+    public function getSubNodeNames(): array {
         return ['name', 'value'];
     }
 
-    public function getType() : string {
+    public function getType(): string {
         return 'Const';
     }
 }

lib/PhpParser/Node/DeclareItem.php

@@ -0,0 +1,37 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\Node;
+use PhpParser\NodeAbstract;
+
+class DeclareItem extends NodeAbstract {
+    /** @var Node\Identifier Key */
+    public $key;
+    /** @var Node\Expr Value */
+    public $value;
+
+    /**
+     * Constructs a declare key=>value pair node.
+     *
+     * @param string|Node\Identifier $key        Key
+     * @param Node\Expr              $value      Value
+     * @param array                  $attributes Additional attributes
+     */
+    public function __construct($key, Node\Expr $value, array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->key = \is_string($key) ? new Node\Identifier($key) : $key;
+        $this->value = $value;
+    }
+
+    public function getSubNodeNames(): array {
+        return ['key', 'value'];
+    }
+
+    public function getType(): string {
+        return 'DeclareItem';
+    }
+}
+
+// @deprecated compatibility alias
+class_alias(DeclareItem::class, Stmt\DeclareDeclare::class);

lib/PhpParser/Node/Expr.php

@@ -4,6 +4,5 @@ namespace PhpParser\Node;
 
 use PhpParser\NodeAbstract;
 
-abstract class Expr extends NodeAbstract
+abstract class Expr extends NodeAbstract {
-{
 }

lib/PhpParser/Node/Expr/ArrayDimFetch.php

@@ -4,8 +4,7 @@ namespace PhpParser\Node\Expr;
 
 use PhpParser\Node\Expr;
 
-class ArrayDimFetch extends Expr
+class ArrayDimFetch extends Expr {
-{
     /** @var Expr Variable */
     public $var;
     /** @var null|Expr Array index / dim */
@@ -18,17 +17,17 @@ class ArrayDimFetch extends Expr
      * @param null|Expr $dim        Array index / dim
      * @param array     $attributes Additional attributes
      */
-    public function __construct(Expr $var, Expr $dim = null, array $attributes = []) {
+    public function __construct(Expr $var, ?Expr $dim = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->dim = $dim;
     }
 
-    public function getSubNodeNames() : array {
+    public function getSubNodeNames(): array {
         return ['var', 'dim'];
     }
 
-    public function getType() : string {
+    public function getType(): string {
         return 'Expr_ArrayDimFetch';
     }
 }

lib/PhpParser/Node/Expr/ArrayItem.php

@@ -1,38 +1,3 @@
 <?php declare(strict_types=1);
 
-namespace PhpParser\Node\Expr;
+require __DIR__ . '/../ArrayItem.php';
-
-use PhpParser\Node\Expr;
-
-class ArrayItem extends Expr
-{
-    /** @var null|Expr Key */
-    public $key;
-    /** @var Expr Value */
-    public $value;
-    /** @var bool Whether to assign by reference */
-    public $byRef;
-
-    /**
-     * Constructs an array item node.
-     *
-     * @param Expr      $value      Value
-     * @param null|Expr $key        Key
-     * @param bool      $byRef      Whether to assign by reference
-     * @param array     $attributes Additional attributes
-     */
-    public function __construct(Expr $value, Expr $key = null, bool $byRef = false, array $attributes = []) {
-        parent::__construct($attributes);
-        $this->key = $key;
-        $this->value = $value;
-        $this->byRef = $byRef;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['key', 'value', 'byRef'];
-    }
-    
-    public function getType() : string {
-        return 'Expr_ArrayItem';
-    }
-}

lib/PhpParser/Node/Expr/Array_.php

@@ -2,13 +2,13 @@
 
 namespace PhpParser\Node\Expr;
 
+use PhpParser\Node\ArrayItem;
 use PhpParser\Node\Expr;
 
-class Array_ extends Expr
+class Array_ extends Expr {
-{
     // For use in "kind" attribute
-    const KIND_LONG = 1;  // array() syntax
+    public const KIND_LONG = 1;  // array() syntax
-    const KIND_SHORT = 2; // [] syntax
+    public const KIND_SHORT = 2; // [] syntax
 
     /** @var ArrayItem[] Items */
     public $items;
@@ -20,15 +20,15 @@ class Array_ extends Expr
      * @param array       $attributes Additional attributes
      */
     public function __construct(array $items = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->items = $items;
     }
 
-    public function getSubNodeNames() : array {
+    public function getSubNodeNames(): array {
         return ['items'];
     }
 
-    public function getType() : string {
+    public function getType(): string {
         return 'Expr_Array';
     }
 }

lib/PhpParser/Node/Expr/ArrowFunction.php

@@ -0,0 +1,78 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node\Expr;
+
+use PhpParser\Node;
+use PhpParser\Node\Expr;
+use PhpParser\Node\FunctionLike;
+
+class ArrowFunction extends Expr implements FunctionLike {
+    /** @var bool */
+    public $static;
+
+    /** @var bool */
+    public $byRef;
+
+    /** @var Node\Param[] */
+    public $params = [];
+
+    /** @var null|Node\Identifier|Node\Name|Node\ComplexType */
+    public $returnType;
+
+    /** @var Expr */
+    public $expr;
+    /** @var Node\AttributeGroup[] */
+    public $attrGroups;
+
+    /**
+     * @param array $subNodes   Array of the following optional subnodes:
+     *                          'static'     => false   : Whether the closure is static
+     *                          'byRef'      => false   : Whether to return by reference
+     *                          'params'     => array() : Parameters
+     *                          'returnType' => null    : Return type
+     *                          'expr'       => Expr    : Expression body
+     *                          'attrGroups' => array() : PHP attribute groups
+     * @param array $attributes Additional attributes
+     */
+    public function __construct(array $subNodes = [], array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->static = $subNodes['static'] ?? false;
+        $this->byRef = $subNodes['byRef'] ?? false;
+        $this->params = $subNodes['params'] ?? [];
+        $returnType = $subNodes['returnType'] ?? null;
+        $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
+        $this->expr = $subNodes['expr'];
+        $this->attrGroups = $subNodes['attrGroups'] ?? [];
+    }
+
+    public function getSubNodeNames(): array {
+        return ['attrGroups', 'static', 'byRef', 'params', 'returnType', 'expr'];
+    }
+
+    public function returnsByRef(): bool {
+        return $this->byRef;
+    }
+
+    public function getParams(): array {
+        return $this->params;
+    }
+
+    public function getReturnType() {
+        return $this->returnType;
+    }
+
+    public function getAttrGroups(): array {
+        return $this->attrGroups;
+    }
+
+    /**
+     * @return Node\Stmt\Return_[]
+     */
+    public function getStmts(): array {
+        return [new Node\Stmt\Return_($this->expr)];
+    }
+
+    public function getType(): string {
+        return 'Expr_ArrowFunction';
+    }
+}

lib/PhpParser/Node/Expr/Assign.php

@@ -4,8 +4,7 @@ namespace PhpParser\Node\Expr;
 
 use PhpParser\Node\Expr;
 
-class Assign extends Expr
+class Assign extends Expr {
-{
     /** @var Expr Variable */
     public $var;
     /** @var Expr Expression */
@@ -19,16 +18,16 @@ class Assign extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }
 
-    public function getSubNodeNames() : array {
+    public function getSubNodeNames(): array {
         return ['var', 'expr'];
     }
 
-    public function getType() : string {
+    public function getType(): string {
         return 'Expr_Assign';
     }
 }

lib/PhpParser/Node/Expr/AssignOp.php

@@ -4,8 +4,7 @@ namespace PhpParser\Node\Expr;
 
 use PhpParser\Node\Expr;
 
-abstract class AssignOp extends Expr
+abstract class AssignOp extends Expr {
-{
     /** @var Expr Variable */
     public $var;
     /** @var Expr Expression */
@@ -19,12 +18,12 @@ abstract class AssignOp extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }
 
-    public function getSubNodeNames() : array {
+    public function getSubNodeNames(): array {
         return ['var', 'expr'];
     }
 }