Release PHP-Parser 4.19.1

Fixes https://github.com/nikic/PHP-Parser/issues/986.

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*.y]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4

.gitattributes

@@ -0,0 +1,11 @@
+/.github         export-ignore
+/doc             export-ignore
+/grammar         export-ignore
+/test            export-ignore
+/test_old        export-ignore
+.editorconfig    export-ignore
+.gitattributes   export-ignore
+.gitignore       export-ignore
+CHANGELOG.md     export-ignore
+phpunit.xml.dist export-ignore
+UPGRADE-*.md     export-ignore

.github/workflows/main.yml

@@ -0,0 +1,88 @@
+# https://help.github.com/en/categories/automating-your-workflow-with-github-actions
+name: Main
+on:
+  push:
+  pull_request:
+
+jobs:
+  tests_71:
+    runs-on: "ubuntu-latest"
+    name: "PHP 7.1 Unit Tests"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v3"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "xdebug"
+          php-version: "7.1"
+          tools: composer:v2
+      - name: "Install dependencies"
+        run: |
+          composer require php-coveralls/php-coveralls:^2.2 --dev --no-update
+          composer update --no-progress --prefer-dist
+      - name: "Tests"
+        run: "php vendor/bin/phpunit --coverage-clover build/logs/clover.xml"
+      - name: Coveralls
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: "php vendor/bin/php-coveralls"
+        if: ${{ success() }}
+  tests:
+    runs-on: "ubuntu-latest"
+    name: "PHP ${{ matrix.php-version }} Unit Tests"
+    strategy:
+      matrix:
+        php-version:
+          - "7.2"
+          - "7.3"
+          - "7.4"
+          - "8.0"
+          - "8.1"
+          - "8.2"
+          - "8.3"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v3"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "${{ matrix.php-version }}"
+          tools: composer:v2
+      - name: "Install dependencies"
+        run: "composer update --no-progress --prefer-dist ${{ matrix.flags }}"
+      - name: "PHPUnit"
+        run: "php vendor/bin/phpunit"
+  test_old_73_80:
+    runs-on: "ubuntu-latest"
+    name: "PHP 7.3 Code on PHP 8.0 Integration Tests"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v3"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "8.0"
+          tools: composer:v2
+      - name: "Install PHP 8 dependencies"
+        run: "composer update --no-progress --prefer-dist"
+      - name: "Tests"
+        run: "test_old/run-php-src.sh 7.3.21"
+  test_old_80_71:
+    runs-on: "ubuntu-latest"
+    name: "PHP 8.1 Code on PHP 7.1 Integration Tests"
+    steps:
+      - name: "Checkout"
+        uses: "actions/checkout@v3"
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@v2"
+        with:
+          coverage: "none"
+          php-version: "7.1"
+          tools: composer:v2
+      - name: "Install PHP 8 dependencies"
+        run: "composer update --no-progress --prefer-dist"
+      - name: "Tests"
+        run: "test_old/run-php-src.sh 8.1.6"

.gitignore

@@ -2,3 +2,4 @@ vendor/
 composer.lock
 grammar/kmyacc.exe
 grammar/y.output
+.phpunit.result.cache

.travis.yml

@@ -1,28 +0,0 @@
-language: php
-dist: trusty
-sudo: false
-
-cache:
-  directories:
-    - $HOME/.composer/cache
-
-php:
-  - 7.0
-  - 7.1
-  - nightly
-
-install:
-  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then composer require satooshi/php-coveralls '~1.0'; fi
-  - composer install --prefer-dist
-
-matrix:
-  allow_failures:
-    - php: nightly
-  fast_finish: true
-
-script:
-  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then vendor/bin/phpunit --coverage-clover build/logs/clover.xml; else vendor/bin/phpunit; fi
-  - if [ $TRAVIS_PHP_VERSION = '7.1' ]; then test_old/run-php-src.sh; fi
-
-after_success:
-  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then php vendor/bin/coveralls; fi

CHANGELOG.md

@@ -1,7 +1,601 @@
-Version 4.0.0-dev
------------------
+Version 4.19.1 (2024-03-17)
+---------------------------
 
-Nothing yet.
+### Fixed
+
+* Fixed "Optional parameter before required parameter" deprecation warning introduced in
+  previous version.
+
+Version 4.19.0 (2024-03-16)
+---------------------------
+
+### Changed
+
+* Do not use implicitly nullable parameters, which are deprecated in PHP 8.4.
+* Remove support for running on PHP 7.0, which does not support explicitly nullable parameters.
+
+Version 4.18.0 (2023-12-10)
+---------------------------
+
+### Added
+
+* Added methods `ParserFactory::createForNewestSupportedVersion()` and
+  `ParserFactory::createForHostVersion()` for forward-compatibility with PHP-Parser 5.0.
+
+### Fixed
+
+* Fixed missing name resolution of class constant types.
+* Fixed class members being dropped if an error is encountered while parsing a later class member
+  (when error recovery is enabeld).
+
+### Changed
+
+* The `grammar/` directory has been excluded from exported git archives.
+
+Version 4.17.1 (2023-08-13)
+---------------------------
+
+### Fixed
+
+* Fixed phpdoc mismatches for `ClassConst::$type` introduced in previous release.
+
+Version 4.17.0 (2023-08-13)
+---------------------------
+
+### Added
+
+* [PHP 8.3] Added support for typed class constants.
+* [PHP 8.3] Added supprot for dynamic class constant fetch.
+* [PHP 8.3] Added support for readonly anonymous classes.
+
+### Fixed
+
+* Fixed missing required parentheses when pretty printing new with an expression class name.
+* Fixed missing required parentheses when pretty printing `(CONST)::$x` and similar.
+
+Version 4.16.0 (2023-06-25)
+---------------------------
+
+### Added
+
+* Added `Name::getParts()` method for forward-compatibility with PHP-Parser 5.
+
+### Deprecated
+
+* Deprecated direct access to `Name::$parts`, which will be removed in PHP-Parser 5.
+
+Version 4.15.5 (2023-05-19)
+---------------------------
+
+### Added
+
+* Added `makePrivate()`, `makeProtected()`, `makePublic()` and `makeReadonly()` methods to
+  `Builder\Param` to allow the creation of promoted parameters.
+
+Version 4.15.4 (2023-03-05)
+---------------------------
+
+### Fixed
+
+* Fixed formatting-preservation for alternative if syntax with trailing comments.
+
+Version 4.15.3 (2023-01-16)
+---------------------------
+
+### Fixed
+
+* Support readonly property with PHP 8.2 DNF type.
+* Fixed PHP attribute group and PHP-Parser attribute mixup in EnumCase builder.
+
+Version 4.15.2 (2022-11-12)
+---------------------------
+
+### Fixed
+
+* Fixed parsing of large hex float literals that contain an "e" character.
+* Fixed tests to pass on 32-bit.
+* Fixed generation of invalid code when using formatting-preserving pretty printer with code that
+  uses inline HTML.
+
+Version 4.15.1 (2022-09-04)
+---------------------------
+
+### Fixed
+
+* Fixed formatting preservation when adding *multiple* attributes to a class/method/etc that
+  previously had none. This fixes a regression in the 4.15.0 release.
+
+Version 4.15.0 (2022-09-03)
+---------------------------
+
+### Added
+
+* PHP 8.2: Added support for `true` type.
+* PHP 8.2: Added support for DNF types.
+
+### Fixed
+
+* Support `readonly` as a function name.
+* Added `__serialize` and `__unserialize` to magic method list.
+* Fixed bounds check in `Name::slice()`.
+* Fixed formatting preservation when adding attributes to a class/method/etc that previously had none.
+
+Version 4.14.0 (2022-05-31)
+---------------------------
+
+### Added
+
+* Added support for readonly classes.
+* Added `rawValue` attribute to `LNumber`, `DNumber` and `String_` nodes, which stores the unparsed
+  value of the literal (e.g. `"1_000"` rather than `1000`).
+
+Version 4.13.2 (2021-11-30)
+---------------------------
+
+### Added
+
+* Added builders for enums and enum cases.
+
+### Fixed
+
+* NullsafeMethodCall now extends from CallLike.
+* The `namespacedName` property populated by the `NameResolver` is now declared on relevant nodes,
+  to avoid a dynamic property deprecation warning with PHP 8.2.
+
+Version 4.13.1 (2021-11-03)
+---------------------------
+
+### Fixed
+
+* Support reserved keywords as enum cases.
+* Support array unpacking in constant expression evaluator.
+
+Version 4.13.0 (2021-09-20)
+---------------------------
+
+### Added
+
+* [PHP 8.1] Added support for intersection types using a new `IntersectionType` node. Additionally
+  a `ComplexType` parent class for `NullableType`, `UnionType` and `IntersectionType` has been
+  added.
+* [PHP 8.1] Added support for explicit octal literals.
+* [PHP 8.1] Added support for first-class callables. These are represented using a call whose first
+  argument is a `VariadicPlaceholder`. The representation is intended to be forward-compatible with
+  partial function application, just like the PHP feature itself. Call nodes now extend from
+  `Expr\CallLike`, which provides an `isFirstClassCallable()` method to determine whether a
+  placeholder id present. `getArgs()` can be used to assert that the call is not a first-class
+  callable and returns `Arg[]` rather than `array<Arg|VariadicPlaceholder>`.
+
+### Fixed
+
+* Multiple modifiers for promoted properties are now accepted. In particular this allows something
+  like `public readonly` for promoted properties.
+* Formatting-preserving pretty printing for comments in array literals has been fixed.
+
+Version 4.12.0 (2021-07-21)
+---------------------------
+
+### Added
+
+* [PHP 8.1] Added support for readonly properties (through a new `MODIFIER_READONLY`).
+* [PHP 8.1] Added support for final class constants.
+
+### Fixed
+
+* Fixed compatibility with PHP 8.1. `&` tokens are now canonicalized to the
+  `T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG` and `T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG` tokens
+  used in PHP 8.1. This happens unconditionally, regardless of whether the emulative lexer is used.
+
+Version 4.11.0 (2021-07-03)
+---------------------------
+
+### Added
+
+* `BuilderFactory::args()` now accepts named arguments.
+* `BuilderFactory::attribute()` has been added.
+* An `addAttribute()` method accepting an `Attribute` or `AttributeGroup` has been adde to all
+  builders that accept attributes, such as `Builder\Class_`.
+
+### Fixed
+
+* `NameResolver` now handles enums.
+* `PrettyPrinter` now prints backing enum type.
+* Builder methods for types now property handle `never` type.
+
+Version 4.10.5 (2021-05-03)
+---------------------------
+
+### Added
+
+* [PHP 8.1] Added support for enums. These are represented using the `Stmt\Enum_` and
+  `Stmt\EnumCase` nodes.
+* [PHP 8.1] Added support for never type. This type will now be returned as an `Identifier` rather
+  than `Name`.
+* Added `ClassConst` builder.
+
+### Changed
+
+* Non-UTF-8 code units in strings will now be hex-encoded.
+
+### Fixed
+
+* Fixed precedence of arrow functions.
+
+Version 4.10.4 (2020-12-20)
+---------------------------
+
+### Fixed
+
+* Fixed position information for variable-variables (#741).
+* Fixed position information for traits/interfaces preceded by if statement (#738).
+
+Version 4.10.3 (2020-12-03)
+---------------------------
+
+### Fixed
+
+* Fixed formatting-preserving pretty printing for `"{$x}"`.
+* Ternary expressions are now treated as non-associative in the pretty printer, in order to
+  generate code that is compatible with the parentheses requirement introduced in PHP 8.
+* Removed no longer necessary `error_clear_last()` call in lexer, which may interfere with fatal
+  error handlers if invoked during shutdown.
+
+
+Version 4.10.2 (2020-09-26)
+------------------
+
+### Fixed
+
+* Fixed check for token emulation conflicts with other libraries.
+
+Version 4.10.1 (2020-09-23)
+---------------------------
+
+### Added
+
+* Added support for recovering from a missing semicolon after a property or class constant
+  declaration.
+
+### Fixed
+
+* Fix spurious whitespace in formatting-preserving pretty printer when both removing and adding
+  elements at the start of a list.
+* Fix incorrect case-sensitivity in keyword token emulation.
+
+Version 4.10.0 (2020-09-19)
+---------------------------
+
+### Added
+
+* [PHP 8.0] Added support for attributes. These are represented using a new `AttributeGroup` node
+  containing `Attribute` nodes. A new `attrGroups` subnode is available on all node types that
+  support attributes, i.e. `Stmt\Class_`, `Stmt\Trait_`, `Stmt\Interface_`, `Stmt\Function_`,
+  `Stmt\ClassMethod`, `Stmt\ClassConst`, `Stmt\Property`, `Expr\Closure`, `Expr\ArrowFunction` and
+  `Param`.
+* [PHP 8.0] Added support for nullsafe properties inside interpolated strings, in line with an
+  upstream change.
+
+### Fixed
+
+* Improved compatibility with other libraries that use forward compatibility defines for PHP tokens.
+
+Version 4.9.1 (2020-08-30)
+--------------------------
+
+### Added
+
+* Added support for removing the first element of a list to the formatting-preserving pretty
+  printer.
+
+### Fixed
+
+* Allow member modifiers as part of namespaced names. These were missed when support for other
+  keywords was added.
+
+Version 4.9.0 (2020-08-18)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for named arguments, represented using a new `name` subnode on `Arg`.
+* [PHP 8.0] Added support for static return type, represented like a normal class return type.
+* [PHP 8.0] Added support for throw expression, represented using a new `Expr\Throw_` node. For
+  backwards compatibility reasons, throw expressions in statement context continue to be
+  represented using `Stmt\Throw_`.
+* [PHP 8.0] Added support for keywords as parts of namespaced names.
+
+### Fixed
+
+* Emit parentheses for class constant fetch with complex left-hand-side.
+* Emit parentheses for new/instanceof on complex class expression.
+
+Version 4.8.0 (2020-08-09)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for nullsafe operator, represented using the new
+  `Expr\NullsafePropertyFetch` and `Expr\NullsafeMethodCall` nodes.
+* Added `phpVersion` option to the emulative lexer, which allows controlling the target version to
+  emulate (defaults to the latest available, currently PHP 8.0). This is useful to parse code that
+  uses reserved keywords from newer PHP versions as identifiers.
+
+Version 4.7.0 (2020-07-25)
+--------------------------
+
+### Added
+
+* Add `ParentConnectingVisitor` and `NodeConnectingVisitor` classes.
+* [PHP 8.0] Added support for match expressions. These are represented using a new `Expr\Match_`
+  containing `MatchArm`s.
+* [PHP 8.0] Added support for trailing comma in closure use lists.
+
+### Fixed
+
+* Fixed missing error for unterminated comment with trailing newline (#688).
+* Compatibility with PHP 8.0 has been restored: Namespaced names are now always represented by
+  `T_NAME_*` tokens, using emulationg on older PHP versions. Full support for reserved keywords
+  in namespaced names is not yet present.
+
+Version 4.6.0 (2020-07-02)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for trailing commas in parameter lists.
+* [PHP 8.0] Added support for constructor promotion. The parameter visibility is stored in
+  `Node\Param::$flags`.
+
+### Fixed
+
+* Comment tokens now always follow the PHP 8 interpretation, and do not include trailing
+  whitespace.
+* As a result of the previous change, some whitespace issues when inserting a statement into a
+  method containing only a comment, and using the formatting-preserving pretty printer, have been
+  resolved.
+
+Version 4.5.0 (2020-06-03)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for the mixed type. This means `mixed` types are now parsed as an
+  `Identifier` rather than a `Name`.
+* [PHP 8.0] Added support for catching without capturing the exception. This means that
+  `Catch_::$var` may now be null.
+
+Version 4.4.0 (2020-04-10)
+--------------------------
+
+### Added
+
+* Added support for passing union types in builders.
+* Added end line, token position and file position information for comments.
+* Added `getProperty()` method to `ClassLike` nodes.
+
+### Fixed
+
+* Fixed generation of invalid code when using the formatting preserving pretty printer, and
+  inserting code next to certain nop statements. The formatting is still ugly though.
+* `getDocComment()` no longer requires that the very last comment before a node be a doc comment.
+  There may not be non-doc comments between the doc comment and the declaration.
+* Allowed arbitrary expressions in `isset()` and `list()`, rather than just variables.
+  In particular, this allows `isset(($x))`, which is legal PHP code.
+* [PHP 8.0] Add support for [variable syntax tweaks RFC](https://wiki.php.net/rfc/variable_syntax_tweaks).
+
+Version 4.3.0 (2019-11-08)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for union types using a new `UnionType` node.
+
+Version 4.2.5 (2019-10-25)
+--------------------------
+
+### Changed
+
+* Tests and documentation are no longer included in source archives. They can still be accessed
+  by cloning the repository.
+* php-yacc is now used to generate the parser. This has no impact on users of the library.
+
+Version 4.2.4 (2019-09-01)
+--------------------------
+
+### Added
+
+* Added getProperties(), getConstants() and getTraitUses() to ClassLike. (#629, #630)
+
+### Fixed
+
+* Fixed flexible heredoc emulation to check for digits after the end label. This synchronizes
+  behavior with the upcoming PHP 7.3.10 release.
+
+Version 4.2.3 (2019-08-12)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for numeric literal separators. (#615)
+
+### Fixed
+
+* Fixed resolution of return types for arrow functions. (#613)
+* Fixed compatibility with PHP 7.4.
+
+Version 4.2.2 (2019-05-25)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for arrow functions using a new `Expr\ArrowFunction` node. (#602)
+* [PHP 7.4] Add support for array spreads, using a new `unpack` subnode on `ArrayItem`. (#609)
+* Added support for inserting into empty list nodes in the formatting preserving pretty printer.
+
+### Changed
+
+* `php-parse` will now print messages to stderr, so that stdout only contains the actual result of
+  the operation (such as a JSON dump). (#605)
+
+### Fixed
+
+* Fixed attribute assignment for zero-length nop statements, and a related assertion failure in
+  the formatting-preserving pretty printer. (#589)
+
+Version 4.2.1 (2019-02-16)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for `??=` operator through a new `AssignOp\Coalesce` node. (#575)
+
+Version 4.2.0 (2019-01-12)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for typed properties through a new `type` subnode of `Stmt\Property`.
+  Additionally `Builder\Property` now has a `setType()` method. (#567)
+* Add `kind` attribute to `Cast\Double_`, which allows to distinguish between `(float)`,
+  `(double)` and `(real)`. The form of the cast will be preserved by the pretty printer. (#565)
+
+### Fixed
+
+* Remove assertion when pretty printing anonymous class with a name (#554).
+
+Version 4.1.1 (2018-12-26)
+--------------------------
+
+### Fixed
+
+* Fix "undefined offset" notice when parsing specific malformed code (#551).
+
+### Added
+
+* Support error recovery for missing return type (`function foo() : {}`) (#544).
+
+Version 4.1.0 (2018-10-10)
+--------------------------
+
+### Added
+
+* Added support for PHP 7.3 flexible heredoc/nowdoc strings, completing support for PHP 7.3. There
+  are two caveats for this feature:
+   * In some rare, pathological cases flexible heredoc/nowdoc strings change the interpretation of
+     existing doc strings. PHP-Parser will now use the new interpretation.
+   * Flexible heredoc/nowdoc strings require special support from the lexer. Because this is not
+     available on PHP versions before 7.3, support has to be emulated. This emulation is not perfect
+     and some cases which we do not expect to occur in practice (such as flexible doc strings being
+     nested within each other through abuse of variable-variable interpolation syntax) may not be
+     recognized correctly.
+* Added `DONT_TRAVERSE_CURRENT_AND_CHILDREN` to `NodeTraverser` to skip both traversal of child
+  nodes, and prevent subsequent visitors from visiting the current node.
+
+Version 4.0.4 (2018-09-18)
+--------------------------
+
+### Added
+
+* The following methods have been added to `BuilderFactory`:
+  * `useTrait()` (fluent builder)
+  * `traitUseAdaptation()` (fluent builder)
+  * `useFunction()` (fluent builder)
+  * `useConst()` (fluent builder)
+  * `var()`
+  * `propertyFetch()`
+  
+### Deprecated
+
+* `Builder\Param::setTypeHint()` has been deprecated in favor of the newly introduced
+  `Builder\Param::setType()`.
+
+Version 4.0.3 (2018-07-15)
+--------------------------
+
+### Fixed
+
+* Fixed possible undefined offset notice in formatting-preserving printer. (#513)
+
+### Added
+
+* Improved error recovery inside arrays.
+* Preserve trailing comment inside classes. **Note:** This change is possibly BC breaking if your
+  code validates that classes can only contain certain statement types. After this change, classes
+  can also contain Nop statements, while this was not previously possible. (#509)
+
+Version 4.0.2 (2018-06-03)
+--------------------------
+
+### Added
+
+* Improved error recovery inside classes.
+* Support error recovery for `foreach` without `as`.
+* Support error recovery for parameters without variable (`function (Type ) {}`).
+* Support error recovery for functions without body (`function ($foo)`).
+
+Version 4.0.1 (2018-03-25)
+--------------------------
+
+### Added
+
+* [PHP 7.3] Added support for trailing commas in function calls.
+* [PHP 7.3] Added support for by-reference array destructuring. 
+* Added checks to node traverser to prevent replacing a statement with an expression or vice versa.
+  This should prevent common mistakes in the implementation of node visitors.
+* Added the following methods to `BuilderFactory`, to simplify creation of expressions:
+  * `funcCall()`
+  * `methodCall()`
+  * `staticCall()`
+  * `new()`
+  * `constFetch()`
+  * `classConstFetch()`
+
+Version 4.0.0 (2018-02-28)
+--------------------------
+
+* No significant code changes since the beta 1 release.
+
+Version 4.0.0-beta1 (2018-01-27)
+--------------------------------
+
+### Fixed
+
+* In formatting-preserving pretty printer: Fixed indentation when inserting into lists. (#466)
+
+### Added
+
+* In formatting-preserving pretty printer: Improved formatting of elements inserted into multi-line
+  arrays.
+
+### Removed
+
+* The `Autoloader` class has been removed. It is now required to use the Composer autoloader.
+
+Version 4.0.0-alpha3 (2017-12-26)
+---------------------------------
+
+### Fixed
+
+* In the formatting-preserving pretty printer:
+  * Fixed comment indentation.
+  * Fixed handling of inline HTML in the fallback case.
+  * Fixed insertion into list nodes that require creation of a code block.
+
+### Added
+
+* Added support for inserting at the start of list nodes in formatting-preserving pretty printer.
+
+Version 4.0.0-alpha2 (2017-11-10)
+---------------------------------
+
+### Added
+
+* In the formatting-preserving pretty printer:
+  * Added support for changing modifiers.
+  * Added support for anonymous classes.
+  * Added support for removing from list nodes.
+  * Improved support for changing comments.
+* Added start token offsets to comments.
 
 Version 4.0.0-alpha1 (2017-10-18)
 ---------------------------------
@@ -22,7 +616,7 @@ Version 4.0.0-alpha1 (2017-10-18)
   AST traversal. This facilitates use in other context, such as class names in doc comments.
   Additionally it provides an API for getting the shortest representation of a name.
 * Added `Node::setAttributes()` method.
-* Added `JsonDecoder`. This allows convertion JSON back into an AST.
+* Added `JsonDecoder`. This allows conversion JSON back into an AST.
 * Added `Name` methods `toLowerString()` and `isSpecialClassName()`.
 * Added `Identifier` and `VarLikeIdentifier` nodes, which are used in place of simple strings in
   many places.
@@ -66,8 +660,31 @@ Version 4.0.0-alpha1 (2017-10-18)
 * The `BuilderAbstract` class has been removed. It's functionality is moved into `BuilderHelpers`.
   However, this is an internal class and should not be used directly.
 
-Version 3.1.2-dev
------------------
+Version 3.1.5 (2018-02-28)
+--------------------------
+
+### Fixed
+
+* Fixed duplicate comment assignment in switch statements. (#469)
+* Improve compatibility with PHP-Scoper. (#477)
+
+Version 3.1.4 (2018-01-25)
+--------------------------
+
+### Fixed
+
+* Fixed pretty printing of `-(-$x)` and `+(+$x)`. (#459)
+
+Version 3.1.3 (2017-12-26)
+--------------------------
+
+### Fixed
+
+* Improve compatibility with php-scoper, by supporting prefixed namespaces in
+  `NodeAbstract::getType()`.
+
+Version 3.1.2 (2017-11-04)
+--------------------------
 
 ### Fixed
 
@@ -77,6 +694,7 @@ Version 3.1.2-dev
 
 * Added `kind` attribute for `Stmt\Namespace_` node, which is one of `KIND_SEMICOLON` or
   `KIND_BRACED`. (#417)
+* Added `setDocComment()` method to namespace builder. (#437)
 
 Version 3.1.1 (2017-09-02)
 --------------------------
@@ -94,7 +712,7 @@ Version 3.1.0 (2017-07-28)
 * [PHP 7.2] Added support for trailing comma in group use statements.
 * [PHP 7.2] Added support for `object` type. This means `object` types will now be represented as a
   builtin type (a simple `"object"` string), rather than a class `Name`.
-  
+
 ### Fixed
 
 * Floating-point numbers are now printed correctly if the LC_NUMERIC locale uses a comma as decimal
@@ -214,7 +832,7 @@ This release primarily improves our support for error recovery.
   `NameResolver::__construct()`.
 * The `NameResolver` now adds a `namespacedName` attribute on name nodes that cannot be statically
   resolved (unqualified unaliased function or constant names in namespaces).
-  
+
 ### Fixed
 
 * Fixed attribute assignment for `GroupUse` prefix and variables in interpolated strings.
@@ -304,7 +922,7 @@ Additionally the following changes were made:
   takes an array of subnodes. Unlike classes/interfaces, traits can only have a `stmts` subnode.
 * The `NodeDumper` now prints class/method/property/constant modifiers, as well as the include and
   use type in a textual representation, instead of only showing the number.
-* All methods on `PrettyPrinter\Standard` are now protected. Previoulsy most of them were public.
+* All methods on `PrettyPrinter\Standard` are now protected. Previously most of them were public.
 
 ### Removed
 
@@ -342,7 +960,7 @@ Version 2.1.0 (2016-04-19)
 * Added `kind` attribute to `Expr\Exit` to distinguish between `exit` and `die`.
 * Added `kind` attribute to `Scalar\LNumber` to distinguish between decimal, binary, octal and
   hexadecimal numbers.
-* Added `kind` attribtue to `Expr\Array` to distinguish between `array()` and `[]`.
+* Added `kind` attribute to `Expr\Array` to distinguish between `array()` and `[]`.
 * Added `kind` attribute to `Scalar\String` and `Scalar\Encapsed` to distinguish between
   single-quoted, double-quoted, heredoc and nowdoc string.
 * Added `docLabel` attribute to `Scalar\String` and `Scalar\Encapsed`, if it is a heredoc or
@@ -458,4 +1076,4 @@ A more detailed description of backwards incompatible changes can be found in th
 
 **This changelog only includes changes from the 2.0 series. For older changes see the
 [1.x series changelog](https://github.com/nikic/PHP-Parser/blob/1.x/CHANGELOG.md) and the
-[0.9 series changelog](https://github.com/nikic/PHP-Parser/blob/0.9/CHANGELOG.md).**
+[0.9 series changelog](https://github.com/nikic/PHP-Parser/blob/0.9/CHANGELOG.md).**

LICENSE

@@ -1,31 +1,29 @@
-Copyright (c) 2011 by Nikita Popov.
+BSD 3-Clause License
 
-Some rights reserved.
+Copyright (c) 2011, Nikita Popov
+All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
+modification, are permitted provided that the following conditions are met:
 
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
 
-    * Redistributions in binary form must reproduce the above
-      copyright notice, this list of conditions and the following
-      disclaimer in the documentation and/or other materials provided
-      with the distribution.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
 
-    * The names of the contributors may not be used to endorse or
-      promote products derived from this software without specific
-      prior written permission.
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1,21 +1,21 @@
 PHP Parser
 ==========
 
-[![Build Status](https://travis-ci.org/nikic/PHP-Parser.svg?branch=master)](https://travis-ci.org/nikic/PHP-Parser) [![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
+[![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
 
-This is a PHP 5.2 to PHP 7.2 parser written in PHP. Its purpose is to simplify static code analysis and
+This is a PHP 5.2 to PHP 8.2 parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
-[**Documentation for version 3.x**][doc_3_x] (stable; for running on PHP >= 5.5; for parsing PHP 5.2 to PHP 7.2).
+[**Documentation for version 4.x**][doc_4_x] (stable; for running on PHP >= 7.1; for parsing PHP 5.2 to PHP 8.2).
 
-[Documentation for version 4.x][doc_master] (development; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 7.2).
+[Documentation for version 3.x][doc_3_x] (unsupported; for running on PHP >= 5.5; for parsing PHP 5.2 to PHP 7.2).
 
 Features
 --------
 
 The main features provided by this library are:
 
- * Parsing PHP 5 and PHP 7 code into an abstract syntax tree (AST).
+ * Parsing PHP 5, PHP 7, and PHP 8 code into an abstract syntax tree (AST).
    * Invalid code can be parsed into a partial AST.
    * The AST contains accurate location information.
  * Dumping the AST in human-readable form.
@@ -183,16 +183,43 @@ Documentation
 
  1. [Introduction](doc/0_Introduction.markdown)
  2. [Usage of basic components](doc/2_Usage_of_basic_components.markdown)
- 3. [Other node tree representations](doc/3_Other_node_tree_representations.markdown)
- 4. [Code generation](doc/4_Code_generation.markdown)
- 5. [Frequently asked questions](doc/5_FAQ.markdown)
 
 Component documentation:
 
+ * [Walking the AST](doc/component/Walking_the_AST.markdown)
+   * Node visitors
+   * Modifying the AST from a visitor
+   * Short-circuiting traversals
+   * Interleaved visitors
+   * Simple node finding API
+   * Parent and sibling references
  * [Name resolution](doc/component/Name_resolution.markdown)
+   * Name resolver options
+   * Name resolution context
  * [Pretty printing](doc/component/Pretty_printing.markdown)
+   * Converting AST back to PHP code
+   * Customizing formatting
+   * Formatting-preserving code transformations
+ * [AST builders](doc/component/AST_builders.markdown)
+   * Fluent builders for AST nodes
  * [Lexer](doc/component/Lexer.markdown)
+   * Lexer options
+   * Token and file positions for nodes
+   * Custom attributes
  * [Error handling](doc/component/Error_handling.markdown)
+   * Column information for errors
+   * Error recovery (parsing of syntactically incorrect code)
+ * [Constant expression evaluation](doc/component/Constant_expression_evaluation.markdown)
+   * Evaluating constant/property/etc initializers
+   * Handling errors and unsupported expressions
+ * [JSON representation](doc/component/JSON_representation.markdown)
+   * JSON encoding and decoding of ASTs
+ * [Performance](doc/component/Performance.markdown)
+   * Disabling Xdebug
+   * Reusing objects
+   * Garbage collection impact
+ * [Frequently asked questions](doc/component/FAQ.markdown)
+   * Parent and sibling references
 
  [doc_3_x]: https://github.com/nikic/PHP-Parser/tree/3.x/doc
- [doc_master]: https://github.com/nikic/PHP-Parser/tree/master/doc
+ [doc_4_x]: https://github.com/nikic/PHP-Parser/tree/4.x/doc

UPGRADE-3.0.md

@@ -4,7 +4,7 @@ Upgrading from PHP-Parser 2.x to 3.0
 The backwards-incompatible changes in this release may be summarized as follows:
 
  * The specific details of the node representation have changed in some cases, primarily to
-   accomodate new PHP 7.1 features.
+   accommodate new PHP 7.1 features.
  * There have been significant changes to the error recovery implementation. This may affect you,
    if you used the error recovery mode or have a custom lexer implementation.
  * A number of deprecated methods were removed.
@@ -115,7 +115,7 @@ public function startLexing($code);
 public function startLexing($code, ErrorHandler $errorHandler = null);
 ```
 
-If you use a custom lexer with overriden `startLexing()` method, it needs to be changed to accept
+If you use a custom lexer with overridden `startLexing()` method, it needs to be changed to accept
 the extra parameter. The value should be passed on to the parent method.
 
 #### Error checks in node constructors
@@ -148,7 +148,7 @@ The following methods, arguments or options have been removed:
    namespace into fully qualified names. For example `foo()` in the global namespace resolves to
    `\foo()`. For names where no static resolution is possible, a `namespacedName` attribute is
    added now, containing the namespaced variant of the name.
- * All methods on `PrettyPrinter\Standard` are now protected. Previoulsy most of them were public.
+ * All methods on `PrettyPrinter\Standard` are now protected. Previously most of them were public.
    The pretty printer should only be invoked using the `prettyPrint()`, `prettyPrintFile()` and
    `prettyPrintExpr()` methods.
  * The node dumper now prints numeric values that act as enums/flags in a string representation.

UPGRADE-4.0.md

@@ -6,7 +6,7 @@ Upgrading from PHP-Parser 3.x to 4.0
 PHP-Parser now requires PHP 7.0 or newer to run. It is however still possible to *parse* PHP 5.2-5.6
 source code, while running on a newer version.
 
-Because HHVM does not support PHP 7, HHVM is no longer supported.
+HHVM is no longer actively supported.
 
 ### Changes to the node structure
 
@@ -17,7 +17,7 @@ Because HHVM does not support PHP 7, HHVM is no longer supported.
   comparisons or strict-mode may require adjustment. The following is an exhaustive list of all
   affected subnodes:
 
-   * `Const::$name`
+   * `Const_::$name`
    * `NullableType::$type` (for simple types)
    * `Param::$type` (for simple types)
    * `Expr\ClassConstFetch::$name`
@@ -29,8 +29,8 @@ Because HHVM does not support PHP 7, HHVM is no longer supported.
    * `Stmt\Class_::$name`
    * `Stmt\ClassMethod::$name`
    * `Stmt\ClassMethod::$returnType` (for simple types)
-   * `Stmt\Function::$name`
-   * `Stmt\Function::$returnType` (for simple types)
+   * `Stmt\Function_::$name`
+   * `Stmt\Function_::$returnType` (for simple types)
    * `Stmt\Goto_::$name`
    * `Stmt\Interface_::$name`
    * `Stmt\Label::$name`
@@ -48,21 +48,21 @@ Because HHVM does not support PHP 7, HHVM is no longer supported.
 * The `name` subnode of `StaticVar` has been renamed to `var` and now contains a `Variable` rather
   than a plain string.
 * The `var` subnode of `ClosureUse` now contains a `Variable` rather than a plain string.
-* The `var` subnode of `Catch` now contains a `Variable` rather than a plain string.
+* The `var` subnode of `Catch_` now contains a `Variable` rather than a plain string.
 * The `alias` subnode of `UseUse` is now `null` if no explicit alias is given. As such,
   `use Foo\Bar` and `use Foo\Bar as Bar` are now represented differently. The `getAlias()` method
   can be used to get the effective alias, even if it is not explicitly given.
-  
+
 ### Miscellaneous
 
-* The indenentation handling in the pretty printer has been changed (this is only relevant if you
+* The indentation handling in the pretty printer has been changed (this is only relevant if you
   extend the pretty printer). Previously indentation was automatic, and parts were excluded using
-  `pNoindent()`. Now no-indent is the default and newlins that require indentation should use
+  `pNoindent()`. Now no-indent is the default and newlines that require indentation should use
   `$this->nl`.
 
 ### Removed functionality
 
-* Removed `type` subnode on `Class`, `ClassMethod` and `Property` nodes. Use `flags` instead.
+* Removed `type` subnode on `Class_`, `ClassMethod` and `Property` nodes. Use `flags` instead.
 * The `ClassConst::isStatic()` method has been removed. Constants cannot have a static modifier.
 * The `NodeTraverser` no longer accepts `false` as a return value from a `leaveNode()` method.
   `NodeTraverser::REMOVE_NODE` should be returned instead.
@@ -73,4 +73,5 @@ Because HHVM does not support PHP 7, HHVM is no longer supported.
 * The XML serializer has been removed. As such, the classes `Serializer\XML`, and
   `Unserializer\XML`, as well as the interfaces `Serializer` and `Unserializer` no longer exist.
 * The `BuilderAbstract` class has been removed. It's functionality is moved into `BuilderHelpers`.
-  However, this is an internal class and should not be used directly.
+  However, this is an internal class and should not be used directly.
+* The `Autoloader` class has been removed in favor of relying on the Composer autoloader.

bin/php-parse

@@ -10,7 +10,7 @@ foreach ([__DIR__ . '/../../../autoload.php', __DIR__ . '/../vendor/autoload.php
 
 ini_set('xdebug.max_nesting_level', 3000);
 
-// Disable XDebug var_dump() output truncation
+// Disable Xdebug var_dump() output truncation
 ini_set('xdebug.var_display_max_children', -1);
 ini_set('xdebug.var_display_max_data', -1);
 ini_set('xdebug.var_display_max_depth', -1);
@@ -45,14 +45,15 @@ $traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);
 foreach ($files as $file) {
     if (strpos($file, '<?php') === 0) {
         $code = $file;
-        echo "====> Code $code\n";
+        fwrite(STDERR, "====> Code $code\n");
     } else {
         if (!file_exists($file)) {
-            die("File $file does not exist.\n");
+            fwrite(STDERR, "File $file does not exist.\n");
+            exit(1);
         }
 
         $code = file_get_contents($file);
-        echo "====> File $file:\n";
+        fwrite(STDERR, "====> File $file:\n");
     }
 
     if ($attributes['with-recovery']) {
@@ -60,7 +61,7 @@ foreach ($files as $file) {
         $stmts = $parser->parse($code, $errorHandler);
         foreach ($errorHandler->getErrors() as $error) {
             $message = formatErrorMessage($error, $code, $attributes['with-column-info']);
-            echo $message . "\n";
+            fwrite(STDERR, $message . "\n");
         }
         if (null === $stmts) {
             continue;
@@ -70,22 +71,26 @@ foreach ($files as $file) {
             $stmts = $parser->parse($code);
         } catch (PhpParser\Error $error) {
             $message = formatErrorMessage($error, $code, $attributes['with-column-info']);
-            die($message . "\n");
+            fwrite(STDERR, $message . "\n");
+            exit(1);
         }
     }
 
     foreach ($operations as $operation) {
         if ('dump' === $operation) {
-            echo "==> Node dump:\n";
+            fwrite(STDERR, "==> Node dump:\n");
             echo $dumper->dump($stmts, $code), "\n";
         } elseif ('pretty-print' === $operation) {
-            echo "==> Pretty print:\n";
+            fwrite(STDERR, "==> Pretty print:\n");
             echo $prettyPrinter->prettyPrintFile($stmts), "\n";
+        } elseif ('json-dump' === $operation) {
+            fwrite(STDERR, "==> JSON dump:\n");
+            echo json_encode($stmts, JSON_PRETTY_PRINT), "\n";
         } elseif ('var-dump' === $operation) {
-            echo "==> var_dump():\n";
+            fwrite(STDERR, "==> var_dump():\n");
             var_dump($stmts);
         } elseif ('resolve-names' === $operation) {
-            echo "==> Resolved names.\n";
+            fwrite(STDERR, "==> Resolved names.\n");
             $stmts = $traverser->traverse($stmts);
         }
     }
@@ -101,9 +106,9 @@ function formatErrorMessage(PhpParser\Error $e, $code, $withColumnInfo) {
 
 function showHelp($error = '') {
     if ($error) {
-        echo $error . "\n\n";
+        fwrite(STDERR, $error . "\n\n");
     }
-    die(<<<OUTPUT
+    fwrite($error ? STDERR : STDOUT, <<<OUTPUT
 Usage: php-parse [operations] file1.php [file2.php ...]
    or: php-parse [operations] "<?php code"
 Turn PHP source code into an abstract syntax tree.
@@ -112,6 +117,7 @@ Operations is a list of the following options (--dump by default):
 
     -d, --dump              Dump nodes using NodeDumper
     -p, --pretty-print      Pretty print file using PrettyPrinter\Standard
+    -j, --json-dump         Print json_encode() result
         --var-dump          var_dump() nodes (for exact structure)
     -N, --resolve-names     Resolve names using NodeVisitor\NameResolver
     -c, --with-column-info  Show column-numbers for errors (if available)
@@ -127,6 +133,7 @@ Example:
 
 OUTPUT
     );
+    exit($error ? 1 : 0);
 }
 
 function parseArgs($args) {
@@ -155,8 +162,9 @@ function parseArgs($args) {
             case '-p':
                 $operations[] = 'pretty-print';
                 break;
-            case '--serialize-xml':
-                $operations[] = 'serialize-xml';
+            case '--json-dump':
+            case '-j':
+                $operations[] = 'json-dump';
                 break;
             case '--var-dump':
                 $operations[] = 'var-dump';

composer.json

@@ -1,8 +1,11 @@
 {
     "name": "nikic/php-parser",
-    "description": "A PHP parser written in PHP",
-    "keywords": ["php", "parser"],
     "type": "library",
+    "description": "A PHP parser written in PHP",
+    "keywords": [
+        "php",
+        "parser"
+    ],
     "license": "BSD-3-Clause",
     "authors": [
         {
@@ -10,21 +13,29 @@
         }
     ],
     "require": {
-        "php": ">=7.0",
+        "php": ">=7.1",
         "ext-tokenizer": "*"
     },
     "require-dev": {
-        "phpunit/phpunit": "^6"
+        "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0",
+        "ircmaxell/php-yacc": "^0.0.7"
+    },
+    "extra": {
+        "branch-alias": {
+            "dev-master": "4.9-dev"
+        }
     },
     "autoload": {
         "psr-4": {
             "PhpParser\\": "lib/PhpParser"
         }
     },
-    "bin": ["bin/php-parse"],
-    "extra": {
-        "branch-alias": {
-            "dev-master": "4.0-dev"
+    "autoload-dev": {
+        "psr-4": {
+            "PhpParser\\": "test/PhpParser/"
         }
-    }
+    },
+    "bin": [
+        "bin/php-parse"
+    ]
 }

doc/0_Introduction.markdown

@@ -1,7 +1,7 @@
 Introduction
 ============
 
-This project is a PHP 5.2 to PHP 7.1 parser **written in PHP itself**.
+This project is a PHP 5.2 to PHP 8.0 parser **written in PHP itself**.
 
 What is this for?
 -----------------
@@ -14,7 +14,7 @@ There are other ways of processing source code. One that PHP supports natively i
 token stream generated by [`token_get_all`][2]. The token stream is much more low level than
 the AST and thus has different applications: It allows to also analyze the exact formatting of
 a file. On the other hand the token stream is much harder to deal with for more complex analysis.
-For example an AST abstracts away the fact that in PHP variables can be written as `$foo`, but also
+For example, an AST abstracts away the fact that, in PHP, variables can be written as `$foo`, but also
 as `$$bar`, `${'foobar'}` or even `${!${''}=barfoo()}`. You don't have to worry about recognizing
 all the different syntaxes from a stream of tokens.
 
@@ -26,17 +26,21 @@ programmatic PHP code analysis are incidentally PHP developers, not C developers
 What can it parse?
 ------------------
 
-The parser supports parsing PHP 5.2-5.6 and PHP 7.
+The parser supports parsing PHP 5.2-8.0, with the following exceptions:
+
+ * Namespaced names containing whitespace (e.g. `Foo \ Bar` instead of `Foo\Bar`) are not supported.
+   These are illegal in PHP 8, but are legal in earlier version. However, PHP-Parser does not
+   support them for any version.
 
 As the parser is based on the tokens returned by `token_get_all` (which is only able to lex the PHP
 version it runs on), additionally a wrapper for emulating tokens from newer versions is provided.
-This allows to parse PHP 7.1 source code running on PHP 5.5, for example. This emulation is somewhat
+This allows to parse PHP 7.4 source code running on PHP 7.1, for example. This emulation is somewhat
 hacky and not perfect, but it should work well on any sane code.
 
 What output does it produce?
 ----------------------------
 
-The parser produces an [Abstract Syntax Tree][1] (AST) also known as a node tree. How this looks like
+The parser produces an [Abstract Syntax Tree][1] (AST) also known as a node tree. How this looks
 can best be seen in an example. The program `<?php echo 'Hi', 'World';` will give you a node tree
 roughly looking like this:
 
@@ -56,7 +60,7 @@ array(
 ```
 
 This matches the structure of the code: An echo statement, which takes two strings as expressions,
-with the values `Hi` and `World!`.
+with the values `Hi` and `World`.
 
 You can also see that the AST does not contain any whitespace information (but most comments are saved).
 So using it for formatting analysis is not possible.

doc/2_Usage_of_basic_components.markdown

@@ -19,7 +19,7 @@ ini_set('xdebug.max_nesting_level', 3000);
 ```
 
 This ensures that there will be no errors when traversing highly nested node trees. However, it is
-preferable to disable XDebug completely, as it can easily make this library more than five times
+preferable to disable Xdebug completely, as it can easily make this library more than five times
 slower.
 
 Parsing
@@ -41,7 +41,7 @@ Kind | Behavior
 `ParserFactory::ONLY_PHP7` | Parse code as PHP 7.
 `ParserFactory::ONLY_PHP5` | Parse code as PHP 5.
 
-Unless you have strong reason to use something else, `PREFER_PHP7` is a reasonable default.
+Unless you have a strong reason to use something else, `PREFER_PHP7` is a reasonable default.
 
 The `create()` method optionally accepts a `Lexer` instance as the second argument. Some use cases
 that require customized lexers are discussed in the [lexer documentation](component/Lexer.markdown).
@@ -50,10 +50,18 @@ Subsequently you can pass PHP code (including the opening `<?php` tag) to the `p
 create a syntax tree. If a syntax error is encountered, an `PhpParser\Error` exception will be thrown:
 
 ```php
+<?php
 use PhpParser\Error;
 use PhpParser\ParserFactory;
 
-$code = '<?php // some code';
+$code = <<<'CODE'
+<?php
+function printLine($msg) {
+    echo $msg, "\n";
+}
+printLine('Hello World!!!');
+CODE;
+
 $parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
 
 try {
@@ -66,27 +74,68 @@ try {
 
 A parser instance can be reused to parse multiple files.
 
-Node tree
----------
+Node dumping
+------------
 
-If you use the above code with `$code = "<?php echo 'Hi ', hi\\getTarget();"` the parser will
-generate a node tree looking like this:
+To dump the abstract syntax tree in human readable form, a `NodeDumper` can be used:
+
+```php
+<?php
+use PhpParser\NodeDumper;
+
+$nodeDumper = new NodeDumper;
+echo $nodeDumper->dump($stmts), "\n";
+```
+
+For the sample code from the previous section, this will produce the following output:
 
 ```
 array(
-    0: Stmt_Echo(
-        exprs: array(
-            0: Scalar_String(
-                value: Hi
+    0: Stmt_Function(
+        byRef: false
+        name: Identifier(
+            name: printLine
+        )
+        params: array(
+            0: Param(
+                type: null
+                byRef: false
+                variadic: false
+                var: Expr_Variable(
+                    name: msg
+                )
+                default: null
             )
-            1: Expr_FuncCall(
-                name: Name(
-                    parts: array(
-                        0: hi
-                        1: getTarget
+        )
+        returnType: null
+        stmts: array(
+            0: Stmt_Echo(
+                exprs: array(
+                    0: Expr_Variable(
+                        name: msg
+                    )
+                    1: Scalar_String(
+                        value:
+
                     )
                 )
-                args: array(
+            )
+        )
+    )
+    1: Stmt_Expression(
+        expr: Expr_FuncCall(
+            name: Name(
+                parts: array(
+                    0: printLine
+                )
+            )
+            args: array(
+                0: Arg(
+                    value: Scalar_String(
+                        value: Hello World!!!
+                    )
+                    byRef: false
+                    unpack: false
                 )
             )
         )
@@ -94,10 +143,30 @@ array(
 )
 ```
 
-Thus `$stmts` will contain an array with only one node, with this node being an instance of
-`PhpParser\Node\Stmt\Echo_`.
+You can also use the `php-parse` script to obtain such a node dump by calling it either with a file
+name or code string:
 
-As PHP is a large language there are approximately 140 different nodes. In order to make work
+```sh
+vendor/bin/php-parse file.php
+vendor/bin/php-parse "<?php foo();"
+```
+
+This can be very helpful if you want to quickly check how certain syntax is represented in the AST.
+
+Node tree structure
+-------------------
+
+Looking at the node dump above, you can see that `$stmts` for this example code is an array of two
+nodes, a `Stmt_Function` and a `Stmt_Expression`. The corresponding class names are:
+
+ * `Stmt_Function -> PhpParser\Node\Stmt\Function_`
+ * `Stmt_Expression -> PhpParser\Node\Stmt\Expression`
+
+The additional `_` at the end of the first class name is necessary, because `Function` is a
+reserved keyword. Many node class names in this library have a trailing `_` to avoid clashing with
+a keyword.
+
+As PHP is a large language there are approximately 140 different nodes. In order to make working
 with them easier they are grouped into three categories:
 
  * `PhpParser\Node\Stmt`s are statement nodes, i.e. language constructs that do not return
@@ -113,8 +182,9 @@ with them easier they are grouped into three categories:
  * There are some nodes not in either of these groups, for example names (`PhpParser\Node\Name`)
    and call arguments (`PhpParser\Node\Arg`).
 
-Some node class names have a trailing `_`. This is used whenever the class name would otherwise clash
-with a PHP keyword.
+The `Node\Stmt\Expression` node is somewhat confusing in that it contains both the terms "statement"
+and "expression". This node distinguishes `expr`, which is a `Node\Expr`, from `expr;`, which is
+an "expression statement" represented by `Node\Stmt\Expression` and containing `expr` as a sub-node.
 
 Every node has a (possibly zero) number of subnodes. You can access subnodes by writing
 `$node->subNodeName`. The `Stmt\Echo_` node has only one subnode `exprs`. So in order to access it
@@ -173,7 +243,7 @@ try {
 
 The above code will output:
 
-    <?php echo 'Hello ', hi\getTarget();
+    echo 'Hello ', hi\getTarget();
 
 As you can see the source code was first parsed using `PhpParser\Parser->parse()`, then changed and then
 again converted to code using `PhpParser\PrettyPrinter\Standard->prettyPrint()`.
@@ -184,6 +254,8 @@ single expression using `prettyPrintExpr()`.
 The `prettyPrintFile()` method can be used to print an entire file. This will include the opening `<?php` tag
 and handle inline HTML as the first/last statement more gracefully.
 
+> Read more: [Pretty printing documentation](component/Pretty_printing.markdown)
+
 Node traversation
 -----------------
 
@@ -267,7 +339,8 @@ All four methods can either return the changed node or not return at all (i.e. `
 case the current node is not changed.
 
 The `enterNode()` method can additionally return the value `NodeTraverser::DONT_TRAVERSE_CHILDREN`,
-which instructs the traverser to skip all children of the current node.
+which instructs the traverser to skip all children of the current node. To furthermore prevent subsequent
+visitors from visiting the current node, `NodeTraverser::DONT_TRAVERSE_CURRENT_AND_CHILDREN` can be used instead.
 
 The `leaveNode()` method can additionally return the value `NodeTraverser::REMOVE_NODE`, in which
 case the current node will be removed from the parent array. Furthermore it is possible to return
@@ -278,10 +351,12 @@ be `array(A, X, Y, Z, C)`.
 Instead of manually implementing the `NodeVisitor` interface you can also extend the `NodeVisitorAbstract`
 class, which will define empty default implementations for all the above methods.
 
+> Read more: [Walking the AST](component/Walking_the_AST.markdown)
+
 The NameResolver node visitor
 -----------------------------
 
-One visitor is already bundled with the package: `PhpParser\NodeVisitor\NameResolver`. This visitor
+One visitor that is already bundled with the package is `PhpParser\NodeVisitor\NameResolver`. This visitor
 helps you work with namespaced code by trying to resolve most names to fully qualified ones.
 
 For example, consider the following code:
@@ -292,7 +367,7 @@ For example, consider the following code:
 In order to know that `B\C` really is `A\C` you would need to track aliases and namespaces yourself.
 The `NameResolver` takes care of that and resolves names as far as possible.
 
-After running it most names will be fully qualified. The only names that will stay unqualified are
+After running it, most names will be fully qualified. The only names that will stay unqualified are
 unqualified function and constant names. These are resolved at runtime and thus the visitor can't
 know which function they are referring to. In most cases this is a non-issue as the global functions
 are meant.
@@ -300,6 +375,8 @@ are meant.
 Also the `NameResolver` adds a `namespacedName` subnode to class, function and constant declarations
 that contains the namespaced name instead of only the shortname that is available via `name`.
 
+> Read more: [Name resolution documentation](component/Name_resolution.markdown)
+
 Example: Converting namespaced code to pseudo namespaces
 --------------------------------------------------------
 
@@ -333,7 +410,7 @@ $files = new \RegexIterator($files, '/\.php$/');
 foreach ($files as $file) {
     try {
         // read the file that should be converted
-        $code = file_get_contents($file);
+        $code = file_get_contents($file->getPathName());
 
         // parse
         $stmts = $parser->parse($code);
@@ -429,7 +506,7 @@ class NodeVisitor_NamespaceConverter extends \PhpParser\NodeVisitorAbstract
             // returning an array merges is into the parent array
             return $node->stmts;
         } elseif ($node instanceof Stmt\Use_) {
-            // return use nodes altogether
+            // remove use nodes altogether
             return NodeTraverser::REMOVE_NODE;
         }
     }

doc/3_Other_node_tree_representations.markdown

@@ -1,228 +0,0 @@
-Other node tree representations
-===============================
-
-It is possible to convert the AST into several textual representations, which serve different uses.
-
-Simple serialization
---------------------
-
-It is possible to serialize the node tree using `serialize()` and also unserialize it using
-`unserialize()`. The output is not human readable and not easily processable from anything
-but PHP, but it is compact and generates quickly. The main application thus is in caching.
-
-Human readable dumping
-----------------------
-
-Furthermore it is possible to dump nodes into a human readable format using the `dump` method of
-`PhpParser\NodeDumper`. This can be used for debugging.
-
-```php
-$code = <<<'CODE'
-<?php
-
-function printLine($msg) {
-    echo $msg, "\n";
-}
-
-printLine('Hello World!!!');
-CODE;
-
-$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::PREFER_PHP7);
-$nodeDumper = new PhpParser\NodeDumper;
-
-try {
-    $stmts = $parser->parse($code);
-
-    echo $nodeDumper->dump($stmts), "\n";
-} catch (PhpParser\Error $e) {
-    echo 'Parse Error: ', $e->getMessage();
-}
-```
-
-The above script will have an output looking roughly like this:
-
-```
-array(
-    0: Stmt_Function(
-        byRef: false
-        params: array(
-            0: Param(
-                name: msg
-                default: null
-                type: null
-                byRef: false
-            )
-        )
-        stmts: array(
-            0: Stmt_Echo(
-                exprs: array(
-                    0: Expr_Variable(
-                        name: msg
-                    )
-                    1: Scalar_String(
-                        value:
-
-                    )
-                )
-            )
-        )
-        name: printLine
-    )
-    1: Expr_FuncCall(
-        name: Name(
-            parts: array(
-                0: printLine
-            )
-        )
-        args: array(
-            0: Arg(
-                value: Scalar_String(
-                    value: Hello World!!!
-                )
-                byRef: false
-            )
-        )
-    )
-)
-```
-
-JSON encoding
--------------
-
-Nodes (and comments) implement the `JsonSerializable` interface. As such, it is possible to JSON
-encode the AST directly using `json_encode()`:
-
-```php
-$code = <<<'CODE'
-<?php
-
-function printLine($msg) {
-    echo $msg, "\n";
-}
-
-printLine('Hello World!!!');
-CODE;
-
-$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::PREFER_PHP7);
-
-try {
-    $stmts = $parser->parse($code);
-
-    echo json_encode($stmts, JSON_PRETTY_PRINT), "\n";
-} catch (PhpParser\Error $e) {
-    echo 'Parse Error: ', $e->getMessage();
-}
-```
-
-This will result in the following output (which includes attributes):
-
-```json
-[
-    {
-        "nodeType": "Stmt_Function",
-        "byRef": false,
-        "name": "printLine",
-        "params": [
-            {
-                "nodeType": "Param",
-                "type": null,
-                "byRef": false,
-                "variadic": false,
-                "name": "msg",
-                "default": null,
-                "attributes": {
-                    "startLine": 3,
-                    "endLine": 3
-                }
-            }
-        ],
-        "returnType": null,
-        "stmts": [
-            {
-                "nodeType": "Stmt_Echo",
-                "exprs": [
-                    {
-                        "nodeType": "Expr_Variable",
-                        "name": "msg",
-                        "attributes": {
-                            "startLine": 4,
-                            "endLine": 4
-                        }
-                    },
-                    {
-                        "nodeType": "Scalar_String",
-                        "value": "\n",
-                        "attributes": {
-                            "startLine": 4,
-                            "endLine": 4,
-                            "kind": 2
-                        }
-                    }
-                ],
-                "attributes": {
-                    "startLine": 4,
-                    "endLine": 4
-                }
-            }
-        ],
-        "attributes": {
-            "startLine": 3,
-            "endLine": 5
-        }
-    },
-    {
-        "nodeType": "Expr_FuncCall",
-        "name": {
-            "nodeType": "Name",
-            "parts": [
-                "printLine"
-            ],
-            "attributes": {
-                "startLine": 7,
-                "endLine": 7
-            }
-        },
-        "args": [
-            {
-                "nodeType": "Arg",
-                "value": {
-                    "nodeType": "Scalar_String",
-                    "value": "Hello World!!!",
-                    "attributes": {
-                        "startLine": 7,
-                        "endLine": 7,
-                        "kind": 1
-                    }
-                },
-                "byRef": false,
-                "unpack": false,
-                "attributes": {
-                    "startLine": 7,
-                    "endLine": 7
-                }
-            }
-        ],
-        "attributes": {
-            "startLine": 7,
-            "endLine": 7
-        }
-    }
-]
-```
-
-The JSON representation may be converted back into a node tree using the `JsonDecoder`:
-
-```php
-<?php
-
-$nodeDecoder = new PhpParser\NodeDecoder();
-$ast = $nodeDecoder->decode($json);
-```
-
-Note that not all ASTs can be represented using JSON. In particular:
-
- * JSON only supports UTF-8 strings.
- * JSON does not support non-finite floating-point numbers. This can occur if the original source
-   code contains non-representable floating-pointing literals such as `1e1000`.
-
-If the node tree is not representable in JSON, the initial `json_encode()` call will fail.

doc/4_Code_generation.markdown

@@ -1,84 +0,0 @@
-Code generation
-===============
-
-It is also possible to generate code using the parser, by first creating an Abstract Syntax Tree and then using the
-pretty printer to convert it to PHP code. To simplify code generation, the project comes with builders which allow
-creating node trees using a fluid interface, instead of instantiating all nodes manually. Builders are available for
-the following syntactic elements:
-
- * namespaces and use statements
- * classes, interfaces and traits
- * methods, functions and parameters
- * properties
-
-Here is an example:
-
-```php
-use PhpParser\BuilderFactory;
-use PhpParser\PrettyPrinter;
-use PhpParser\Node;
-
-$factory = new BuilderFactory;
-$node = $factory->namespace('Name\Space')
-    ->addStmt($factory->use('Some\Other\Thingy')->as('SomeOtherClass'))
-    ->addStmt($factory->class('SomeOtherClass')
-        ->extend('SomeClass')
-        ->implement('A\Few', '\Interfaces')
-        ->makeAbstract() // ->makeFinal()
-
-        ->addStmt($factory->method('someMethod')
-            ->makePublic()
-            ->makeAbstract() // ->makeFinal()
-            ->setReturnType('bool')
-            ->addParam($factory->param('someParam')->setTypeHint('SomeClass'))
-            ->setDocComment('/**
-                              * This method does something.
-                              *
-                              * @param SomeClass And takes a parameter
-                              */')
-        )
-
-        ->addStmt($factory->method('anotherMethod')
-            ->makeProtected() // ->makePublic() [default], ->makePrivate()
-            ->addParam($factory->param('someParam')->setDefault('test'))
-            // it is possible to add manually created nodes
-            ->addStmt(new Node\Expr\Print_(new Node\Expr\Variable('someParam')))
-        )
-
-        // properties will be correctly reordered above the methods
-        ->addStmt($factory->property('someProperty')->makeProtected())
-        ->addStmt($factory->property('anotherProperty')->makePrivate()->setDefault(array(1, 2, 3)))
-    )
-
-    ->getNode()
-;
-
-$stmts = array($node);
-$prettyPrinter = new PrettyPrinter\Standard();
-echo $prettyPrinter->prettyPrintFile($stmts);
-```
-
-This will produce the following output with the standard pretty printer:
-
-```php
-<?php
-
-namespace Name\Space;
-
-use Some\Other\Thingy as SomeClass;
-abstract class SomeOtherClass extends SomeClass implements A\Few, \Interfaces
-{
-    protected $someProperty;
-    private $anotherProperty = array(1, 2, 3);
-    /**
-     * This method does something.
-     *
-     * @param SomeClass And takes a parameter
-     */
-    public abstract function someMethod(SomeClass $someParam) : bool;
-    protected function anotherMethod($someParam = 'test')
-    {
-        print $someParam;
-    }
-}
-```

doc/5_FAQ.markdown

@@ -1,68 +0,0 @@
-Frequently Asked Questions
-==========================
-
- * [How can the parent of a node be obtained?](#how-can-the-parent-of-a-node-be-obtained)
- * [How can the next/previous sibling of a node be obtained?](#how-can-the-nextprevious-sibling-of-a-node-be-obtained)
-
-How can the parent of a node be obtained?
------
-
-The AST does not store parent nodes by default. However, it is easy to add a custom parent node
-attribute using a custom node visitor:
-
-```php
-use PhpParser\Node;
-use PhpParser\NodeVisitorAbstract;
-
-class ParentConnector extends NodeVisitorAbstract {
-    private $stack;
-    public function beforeTraverse(array $nodes) {
-        $this->stack = [];
-    }
-    public function enterNode(Node $node) {
-        if (!empty($this->stack)) {
-            $node->setAttribute('parent', $this->stack[count($this->stack)-1]);
-        }
-        $this->stack[] = $node;
-    }
-    public function leaveNode(Node $node) {
-        array_pop($this->stack);
-    }
-}
-```
-
-After running this visitor, the parent node can be obtained through `$node->getAttribute('parent')`.
-
-How can the next/previous sibling of a node be obtained?
------
-
-Again, siblings are not stored by default, but the visitor from the previous entry can be easily
-extended to store the previous / next node with a common parent as well:
-
-```php
-use PhpParser\Node;
-use PhpParser\NodeVisitorAbstract;
-
-class NodeConnector extends NodeVisitorAbstract {
-    private $stack;
-    private $prev;
-    public function beforeTraverse(array $nodes) {
-        $this->stack = [];
-        $this->prev = null;
-    }
-    public function enterNode(Node $node) {
-        if (!empty($this->stack)) {
-            $node->setAttribute('parent', $this->stack[count($this->stack)-1]);
-        }
-        if ($this->prev && $this->prev->getAttribute('parent') == $node->getAttribute('parent')) {
-            $node->setAttribute('prev', $this->prev);
-            $this->prev->setAttribute('next', $node);
-        }
-        $this->stack[] = $node;
-    }
-    public function leaveNode(Node $node) {
-        $this->prev = $node;
-        array_pop($this->stack);
-    }
-}
-```

doc/README.md

@@ -6,13 +6,17 @@ Guide
 
   1. [Introduction](0_Introduction.markdown)
   2. [Usage of basic components](2_Usage_of_basic_components.markdown)
-  3. [Other node tree representations](3_Other_node_tree_representations.markdown)
-  4. [Code generation](4_Code_generation.markdown)
-  5. [Frequently asked questions](5_FAQ.markdown)
- 
+
 Component documentation
 -----------------------
- 
+
+  * [Walking the AST](component/Walking_the_AST.markdown)
+    * Node visitors
+    * Modifying the AST from a visitor
+    * Short-circuiting traversals
+    * Interleaved visitors
+    * Simple node finding API
+    * Parent and sibling references
   * [Name resolution](component/Name_resolution.markdown)
     * Name resolver options
     * Name resolution context
@@ -20,6 +24,8 @@ Component documentation
     * Converting AST back to PHP code
     * Customizing formatting
     * Formatting-preserving code transformations
+  * [AST builders](component/AST_builders.markdown)
+    * Fluent builders for AST nodes
   * [Lexer](component/Lexer.markdown)
     * Lexer options
     * Token and file positions for nodes
@@ -27,3 +33,14 @@ Component documentation
   * [Error handling](component/Error_handling.markdown)
     * Column information for errors
     * Error recovery (parsing of syntactically incorrect code)
+  * [Constant expression evaluation](component/Constant_expression_evaluation.markdown)
+    * Evaluating constant/property/etc initializers
+    * Handling errors and unsupported expressions
+  * [JSON representation](component/JSON_representation.markdown)
+    * JSON encoding and decoding of ASTs
+  * [Performance](component/Performance.markdown)
+    * Disabling Xdebug
+    * Reusing objects
+    * Garbage collection impact
+  * [Frequently asked questions](component/FAQ.markdown)
+    * Parent and sibling references

doc/component/AST_builders.markdown

@@ -0,0 +1,138 @@
+AST builders
+============
+
+When PHP-Parser is used to generate (or modify) code by first creating an Abstract Syntax Tree and
+then using the [pretty printer](Pretty_printing.markdown) to convert it to PHP code, it can often
+be tedious to manually construct AST nodes. The project provides a number of utilities to simplify
+the construction of common AST nodes.
+
+Fluent builders
+---------------
+
+The library comes with a number of builders, which allow creating node trees using a fluent
+interface. Builders are created using the `BuilderFactory` and the final constructed node is
+accessed through `getNode()`. Fluent builders are available for
+the following syntactic elements:
+
+ * namespaces and use statements
+ * classes, interfaces and traits
+ * methods, functions and parameters
+ * properties
+
+Here is an example:
+
+```php
+use PhpParser\BuilderFactory;
+use PhpParser\PrettyPrinter;
+use PhpParser\Node;
+
+$factory = new BuilderFactory;
+$node = $factory->namespace('Name\Space')
+    ->addStmt($factory->use('Some\Other\Thingy')->as('SomeClass'))
+    ->addStmt($factory->useFunction('strlen'))
+    ->addStmt($factory->useConst('PHP_VERSION'))
+    ->addStmt($factory->class('SomeOtherClass')
+        ->extend('SomeClass')
+        ->implement('A\Few', '\Interfaces')
+        ->makeAbstract() // ->makeFinal()
+
+        ->addStmt($factory->useTrait('FirstTrait'))
+
+        ->addStmt($factory->useTrait('SecondTrait', 'ThirdTrait')
+            ->and('AnotherTrait')
+            ->with($factory->traitUseAdaptation('foo')->as('bar'))
+            ->with($factory->traitUseAdaptation('AnotherTrait', 'baz')->as('test'))
+            ->with($factory->traitUseAdaptation('AnotherTrait', 'func')->insteadof('SecondTrait')))
+
+        ->addStmt($factory->method('someMethod')
+            ->makePublic()
+            ->makeAbstract() // ->makeFinal()
+            ->setReturnType('bool') // ->makeReturnByRef()
+            ->addParam($factory->param('someParam')->setType('SomeClass'))
+            ->setDocComment('/**
+                              * This method does something.
+                              *
+                              * @param SomeClass And takes a parameter
+                              */')
+        )
+
+        ->addStmt($factory->method('anotherMethod')
+            ->makeProtected() // ->makePublic() [default], ->makePrivate()
+            ->addParam($factory->param('someParam')->setDefault('test'))
+            // it is possible to add manually created nodes
+            ->addStmt(new Node\Expr\Print_(new Node\Expr\Variable('someParam')))
+        )
+
+        // properties will be correctly reordered above the methods
+        ->addStmt($factory->property('someProperty')->makeProtected())
+        ->addStmt($factory->property('anotherProperty')->makePrivate()->setDefault(array(1, 2, 3)))
+    )
+
+    ->getNode()
+;
+
+$stmts = array($node);
+$prettyPrinter = new PrettyPrinter\Standard();
+echo $prettyPrinter->prettyPrintFile($stmts);
+```
+
+This will produce the following output with the standard pretty printer:
+
+```php
+<?php
+
+namespace Name\Space;
+
+use Some\Other\Thingy as SomeClass;
+use function strlen;
+use const PHP_VERSION;
+abstract class SomeOtherClass extends SomeClass implements A\Few, \Interfaces
+{
+    use FirstTrait;
+    use SecondTrait, ThirdTrait, AnotherTrait {
+        foo as bar;
+        AnotherTrait::baz as test;
+        AnotherTrait::func insteadof SecondTrait;
+    }
+    protected $someProperty;
+    private $anotherProperty = array(1, 2, 3);
+    /**
+     * This method does something.
+     *
+     * @param SomeClass And takes a parameter
+     */
+    public abstract function someMethod(SomeClass $someParam) : bool;
+    protected function anotherMethod($someParam = 'test')
+    {
+        print $someParam;
+    }
+}
+```
+
+Additional helper methods
+-------------------------
+
+The `BuilderFactory` also provides a number of additional helper methods, which directly return
+nodes. The following methods are currently available:
+
+ * `val($value)`: Creates an AST node for a literal value like `42` or `[1, 2, 3]`.
+ * `var($name)`: Creates variable node.
+ * `args(array $args)`: Creates an array of function/method arguments, including the required `Arg`
+   wrappers. Also converts literals to AST nodes.
+ * `funcCall($name, array $args = [])`: Create a function call node. Converts `$name` to a `Name`
+   node and normalizes arguments.
+ * `methodCall(Expr $var, $name, array $args = [])`: Create a method call node. Converts `$name` to
+   an `Identifier` node and normalizes arguments.
+ * `staticCall($class, $name, array $args = [])`: Create a static method call node. Converts
+   `$class` to a `Name` node, `$name` to an `Identifier` node and normalizes arguments.
+ * `new($class, array $args = [])`: Create a "new" (object creation) node. Converts `$class` to a
+   `Name` node.
+ * `constFetch($name)`: Create a constant fetch node. Converts `$name` to a `Name` node.
+ * `classConstFetch($class, $name)`: Create a class constant fetch node. Converts `$class` to a
+   `Name` node and `$name` to an `Identifier` node.
+ * `propertyFetch($var, $name)`: Creates a property fetch node. Converts `$name` to an `Identifier`
+   node.
+ * `concat(...$exprs)`: Create a tree of `BinaryOp\Concat` nodes for the given expressions.
+
+These methods may be expanded on an as-needed basis. Please open an issue or PR if a common
+operation is missing.

doc/component/Constant_expression_evaluation.markdown

@@ -0,0 +1,115 @@
+Constant expression evaluation
+==============================
+
+Initializers for constants, properties, parameters, etc. have limited support for expressions. For
+example:
+
+```php
+<?php
+class Test {
+    const SECONDS_IN_HOUR = 60 * 60;
+    const SECONDS_IN_DAY = 24 * self::SECONDS_IN_HOUR;
+}
+```
+
+PHP-Parser supports evaluation of such constant expressions through the `ConstExprEvaluator` class:
+
+```php
+<?php
+
+use PhpParser\{ConstExprEvaluator, ConstExprEvaluationException};
+
+$evalutator = new ConstExprEvaluator();
+try {
+    $value = $evalutator->evaluateSilently($someExpr);
+} catch (ConstExprEvaluationException $e) {
+    // Either the expression contains unsupported expression types,
+    // or an error occurred during evaluation
+}
+```
+
+Error handling
+--------------
+
+The constant evaluator provides two methods, `evaluateDirectly()` and `evaluateSilently()`, which
+differ in error behavior. `evaluateDirectly()` will evaluate the expression as PHP would, including
+any generated warnings or Errors. `evaluateSilently()` will instead convert warnings and Errors into
+a `ConstExprEvaluationException`. For example:
+
+```php
+<?php
+
+use PhpParser\{ConstExprEvaluator, ConstExprEvaluationException};
+use PhpParser\Node\{Expr, Scalar};
+
+$evaluator = new ConstExprEvaluator();
+
+// 10 / 0
+$expr = new Expr\BinaryOp\Div(new Scalar\LNumber(10), new Scalar\LNumber(0));
+
+var_dump($evaluator->evaluateDirectly($expr)); // float(INF)
+// Warning: Division by zero
+
+try {
+    $evaluator->evaluateSilently($expr);
+} catch (ConstExprEvaluationException $e) {
+    var_dump($e->getPrevious()->getMessage()); // Division by zero
+}
+```
+
+For the purposes of static analysis, you will likely want to use `evaluateSilently()` and leave
+erroring expressions unevaluated.
+
+Unsupported expressions and evaluator fallback
+----------------------------------------------
+
+The constant expression evaluator supports all expression types that are permitted in constant
+expressions, apart from the following:
+
+ * `Scalar\MagicConst\*`
+ * `Expr\ConstFetch` (only null/false/true are handled)
+ * `Expr\ClassConstFetch`
+
+Handling these expression types requires non-local information, such as which global constants are
+defined. By default, the evaluator will throw a `ConstExprEvaluationException` when it encounters
+an unsupported expression type.
+
+It is possible to override this behavior and support resolution for these expression types by
+specifying an evaluation fallback function:
+
+```php
+<?php
+
+use PhpParser\{ConstExprEvaluator, ConstExprEvaluationException};
+use PhpParser\Node\Expr;
+
+$evalutator = new ConstExprEvaluator(function(Expr $expr) {
+    if ($expr instanceof Expr\ConstFetch) {
+        return fetchConstantSomehow($expr);
+    }
+    if ($expr instanceof Expr\ClassConstFetch) {
+        return fetchClassConstantSomehow($expr);
+    }
+    // etc.
+    throw new ConstExprEvaluationException(
+        "Expression of type {$expr->getType()} cannot be evaluated");
+});
+
+try {
+    $evalutator->evaluateSilently($someExpr);
+} catch (ConstExprEvaluationException $e) {
+    // Handle exception
+}
+```
+
+Implementers are advised to ensure that evaluation of indirect constant references cannot lead to
+infinite recursion. For example, the following code could lead to infinite recursion if constant
+lookup is implemented naively.
+
+```php
+<?php
+class Test {
+    const A = self::B;
+    const B = self::A;
+}
+```

doc/component/Error_handling.markdown

@@ -27,7 +27,7 @@ try {
 }
 ```
 
-Before using column information its availability needs to be checked with `$e->hasColumnInfo()`, as the precise
+Before using column information, its availability needs to be checked with `$e->hasColumnInfo()`, as the precise
 location of an error cannot always be determined. The methods for retrieving column information also have to be passed
 the source code of the parsed file. An example for printing an error:
 
@@ -36,7 +36,7 @@ if ($e->hasColumnInfo()) {
     echo $e->getRawMessage() . ' from ' . $e->getStartLine() . ':' . $e->getStartColumn($code)
         . ' to ' . $e->getEndLine() . ':' . $e->getEndColumn($code);
     // or:
-    echo $e->getMessageWithColumnInfo();
+    echo $e->getMessageWithColumnInfo($code);
 } else {
     echo $e->getMessage();
 }
@@ -72,4 +72,4 @@ if (null !== $stmts) {
 }
 ```
 
-The `NameResolver` visitor also accepts an `ErrorHandler` as a constructor argument.
+The `NameResolver` visitor also accepts an `ErrorHandler` as a constructor argument.

doc/component/FAQ.markdown

@@ -0,0 +1,55 @@
+Frequently Asked Questions
+==========================
+
+ * [How can the parent of a node be obtained?](#how-can-the-parent-of-a-node-be-obtained)
+ * [How can the next/previous sibling of a node be obtained?](#how-can-the-nextprevious-sibling-of-a-node-be-obtained)
+
+How can the parent of a node be obtained?
+-----
+
+The AST does not store parent nodes by default. However, the `ParentConnectingVisitor` can be used to achieve this:
+
+```php
+use PhpParser\NodeTraverser;
+use PhpParser\NodeVisitor\ParentConnectingVisitor;
+use PhpParser\ParserFactory;
+
+$code = '...';
+
+$traverser = new NodeTraverser;
+$traverser->addVisitor(new ParentConnectingVisitor);
+
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+$ast    = $parser->parse($code);
+$ast    = $traverser->traverse($ast);
+```
+
+After running this visitor, the parent node can be obtained through `$node->getAttribute('parent')`.
+
+How can the next/previous sibling of a node be obtained?
+-----
+
+Again, siblings are not stored by default, but the `NodeConnectingVisitor` can be used to store
+the previous / next node with a common parent as well:
+
+```php
+use PhpParser\NodeTraverser;
+use PhpParser\NodeVisitor\NodeConnectingVisitor;
+use PhpParser\ParserFactory;
+
+$code = '...';
+
+$traverser = new NodeTraverser;
+$traverser->addVisitor(new NodeConnectingVisitor);
+
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+$ast    = $parser->parse($code);
+$ast    = $traverser->traverse($ast);
+```
+
+After running this visitor, the parent node can be obtained through `$node->getAttribute('parent')`,
+the previous node can be obtained through `$node->getAttribute('previous')`, and the next node can be
+obtained through `$node->getAttribute('next')`.
+
+`ParentConnectingVisitor` and `NodeConnectingVisitor` should not be used at the same time. The latter
+includes the functionality of the former.

doc/component/JSON_representation.markdown

@@ -0,0 +1,131 @@
+JSON representation
+===================
+
+Nodes (and comments) implement the `JsonSerializable` interface. As such, it is possible to JSON
+encode the AST directly using `json_encode()`:
+
+```php
+<?php
+
+use PhpParser\ParserFactory;
+
+$code = <<<'CODE'
+<?php
+
+/** @param string $msg */
+function printLine($msg) {
+    echo $msg, "\n";
+}
+CODE;
+
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
+
+try {
+    $stmts = $parser->parse($code);
+
+    echo json_encode($stmts, JSON_PRETTY_PRINT), "\n";
+} catch (PhpParser\Error $e) {
+    echo 'Parse Error: ', $e->getMessage();
+}
+```
+
+This will result in the following output (which includes attributes):
+
+```json
+[
+    {
+        "nodeType": "Stmt_Function",
+        "byRef": false,
+        "name": {
+            "nodeType": "Identifier",
+            "name": "printLine",
+            "attributes": {
+                "startLine": 4,
+                "endLine": 4
+            }
+        },
+        "params": [
+            {
+                "nodeType": "Param",
+                "type": null,
+                "byRef": false,
+                "variadic": false,
+                "var": {
+                    "nodeType": "Expr_Variable",
+                    "name": "msg",
+                    "attributes": {
+                        "startLine": 4,
+                        "endLine": 4
+                    }
+                },
+                "default": null,
+                "attributes": {
+                    "startLine": 4,
+                    "endLine": 4
+                }
+            }
+        ],
+        "returnType": null,
+        "stmts": [
+            {
+                "nodeType": "Stmt_Echo",
+                "exprs": [
+                    {
+                        "nodeType": "Expr_Variable",
+                        "name": "msg",
+                        "attributes": {
+                            "startLine": 5,
+                            "endLine": 5
+                        }
+                    },
+                    {
+                        "nodeType": "Scalar_String",
+                        "value": "\n",
+                        "attributes": {
+                            "startLine": 5,
+                            "endLine": 5,
+                            "kind": 2
+                        }
+                    }
+                ],
+                "attributes": {
+                    "startLine": 5,
+                    "endLine": 5
+                }
+            }
+        ],
+        "attributes": {
+            "startLine": 4,
+            "comments": [
+                {
+                    "nodeType": "Comment_Doc",
+                    "text": "\/** @param string $msg *\/",
+                    "line": 3,
+                    "filePos": 9,
+                    "tokenPos": 2
+                }
+            ],
+            "endLine": 6
+        }
+    }
+]
+```
+
+The JSON representation may be converted back into an AST using the `JsonDecoder`:
+
+```php
+<?php
+
+$jsonDecoder = new PhpParser\JsonDecoder();
+$ast = $jsonDecoder->decode($json);
+```
+
+Note that not all ASTs can be represented using JSON. In particular:
+
+ * JSON only supports UTF-8 strings.
+ * JSON does not support non-finite floating-point numbers. This can occur if the original source
+   code contains non-representable floating-pointing literals such as `1e1000`.
+
+If the node tree is not representable in JSON, the initial `json_encode()` call will fail.
+
+From the command line, a JSON dump can be obtained using `vendor/bin/php-parse -j file.php`.

doc/component/Lexer.markdown

@@ -107,9 +107,9 @@ function handleHaltCompiler(): string;
 function getNextToken(string &$value = null, array &$startAttributes = null, array &$endAttributes = null): int;
 ```
 
-The `startLexing()` method is invoked with the source code that is to be lexed (including the opening tag) whenever the
-`parse()` method of the parser is called. It can be used to reset state or preprocess the source code or tokens. The
-passes `ErrorHandler` should be used to report lexing errors.
+The `startLexing()` method is invoked whenever the `parse()` method of the parser is called and is passed the source
+code that is to be lexed (including the opening tag). It can be used to reset state or preprocess the source code or tokens. The
+passed `ErrorHandler` should be used to report lexing errors.
 
 The `getTokens()` method returns the current token array, in the usual `token_get_all()` format. This method is not
 used by the parser (which uses `getNextToken()`), but is useful in combination with the token position attributes.

doc/component/Performance.markdown

@@ -0,0 +1,65 @@
+Performance
+===========
+
+Parsing is computationally expensive task, to which the PHP language is not very well suited.
+Nonetheless, there are a few things you can do to improve the performance of this library, which are
+described in the following.
+
+Xdebug
+------
+
+Running PHP with Xdebug adds a lot of overhead, especially for code that performs many method calls.
+Just by loading Xdebug (without enabling profiling or other more intrusive Xdebug features), you
+can expect that code using PHP-Parser will be approximately *five times slower*.
+
+As such, you should make sure that Xdebug is not loaded when using this library. Note that setting
+the `xdebug.default_enable=0` ini option does *not* disable Xdebug. The *only* way to disable
+Xdebug is to not load the extension in the first place.
+
+If you are building a command-line utility for use by developers (who often have Xdebug enabled),
+you may want to consider automatically restarting PHP with Xdebug unloaded. The
+[composer/xdebug-handler](https://github.com/composer/xdebug-handler) package can be used to do
+this.
+
+If you do run with Xdebug, you may need to increase the `xdebug.max_nesting_level` option to a
+higher level, such as 3000. While the parser itself is recursion free, most other code working on
+the AST uses recursion and will generate an error if the value of this option is too low.
+
+Assertions
+----------
+
+Assertions should be disabled in a production context by setting `zend.assertions=-1` (or
+`zend.assertions=0` if set at runtime). The library currently doesn't make heavy use of assertions,
+but they are used in an increasing number of places.
+
+Object reuse
+------------
+
+Many objects in this project are designed for reuse. For example, one `Parser` object can be used to
+parse multiple files.
+
+When possible, objects should be reused rather than being newly instantiated for every use. Some
+objects have expensive initialization procedures, which will be unnecessarily repeated if the object
+is not reused. (Currently two objects with particularly expensive setup are lexers and pretty
+printers, though the details might change between versions of this library.)
+
+Garbage collection
+------------------
+
+A limitation in PHP's cyclic garbage collector may lead to major performance degradation when the
+active working set exceeds 10000 objects (or arrays). Especially when parsing very large files this
+limit is significantly exceeded and PHP will spend the majority of time performing unnecessary
+garbage collection attempts.
+
+Without GC, parsing time is roughly linear in the input size. With GC, this degenerates to quadratic
+runtime for large files. While the specifics may differ, as a rough guideline you may expect a 2.5x
+GC overhead for 500KB files and a 5x overhead for 1MB files.
+
+Because this a limitation in PHP's implementation, there is no easy way to work around this. If
+possible, you should avoid parsing very large files, as they will impact overall execution time
+disproportionally (and are usually generated anyway).
+
+Of course, you can also try to (temporarily) disable GC. By design the AST generated by PHP-Parser
+is cycle-free, so the AST itself will never cause leaks with GC disabled. However, other code
+(including for example the parser object itself) may hold cycles, so disabling of GC should be
+approached with care.

doc/component/Pretty_printing.markdown

@@ -17,7 +17,7 @@ $stmts = $parser->parse($code);
 // MODIFY $stmts here
 
 $prettyPrinter = new PhpParser\PrettyPrinter\Standard;
-$newCode = $prettyPrinter->prettyPrintFile();
+$newCode = $prettyPrinter->prettyPrintFile($stmts);
 ```
 
 The pretty printer has three basic printing methods: `prettyPrint()`, `prettyPrintFile()` and
@@ -50,14 +50,13 @@ Formatting-preserving pretty printing
 
 For automated code refactoring, migration and similar, you will usually only want to modify a small
 portion of the code and leave the remainder alone. The basic pretty printer is not suitable for
-this, because it will also reformat parts of the code, which have not been modified.
+this, because it will also reformat parts of the code which have not been modified.
 
-Since PHP-Parser 4.0 an experimental formatting-preserving pretty-printing mode is available, which
-attempts to preserve the formatting of code, those AST nodes have not changed, and only reformat
+Since PHP-Parser 4.0, an experimental formatting-preserving pretty-printing mode is available, which
+attempts to preserve the formatting of code (those AST nodes that have not changed) and only reformat
 code which has been modified or newly inserted.
 
-Use of the formatting-preservation functionality currently requires some additional preparatory
-steps:
+Use of the formatting-preservation functionality requires some additional preparatory steps:
 
 ```php
 use PhpParser\{Lexer, NodeTraverser, NodeVisitor, Parser, PrettyPrinter};
@@ -86,7 +85,12 @@ $newStmts = $traverser->traverse($oldStmts);
 $newCode = $printer->printFormatPreserving($newStmts, $oldStmts, $oldTokens);
 ```
 
+If you make use of the name resolution functionality, you will likely want to disable the
+`replaceNodes` option. This will add resolved names as attributes, instead of directlying modifying
+the AST and causing spurious changes to the pretty printed code. For more information, see the
+[name resolution documentation](Name_resolution.markdown).
+
 This functionality is experimental and not yet fully implemented. It should not provide incorrect
 code, but it may sometimes reformat more code than necessary. Open issues are tracked in
 [issue #344](https://github.com/nikic/PHP-Parser/issues/344). If you encounter problems while using
-this functionality, please open an issue, so we know what to prioritize.
+this functionality, please open an issue, so we know what to prioritize.

doc/component/Walking_the_AST.markdown

@@ -0,0 +1,337 @@
+Walking the AST
+===============
+
+The most common way to work with the AST is by using a node traverser and one or more node visitors.
+As a basic example, the following code changes all literal integers in the AST into strings (e.g.,
+`42` becomes `'42'`.)
+
+```php
+use PhpParser\{Node, NodeTraverser, NodeVisitorAbstract};
+
+$traverser = new NodeTraverser;
+$traverser->addVisitor(new class extends NodeVisitorAbstract {
+    public function leaveNode(Node $node) {
+        if ($node instanceof Node\Scalar\LNumber) {
+            return new Node\Scalar\String_((string) $node->value);
+        }
+    }
+});
+
+$stmts = ...;
+$modifiedStmts = $traverser->traverse($stmts);
+```
+
+Node visitors
+-------------
+
+Each node visitor implements an interface with following four methods:
+
+```php
+interface NodeVisitor {
+    public function beforeTraverse(array $nodes);
+    public function enterNode(Node $node);
+    public function leaveNode(Node $node);
+    public function afterTraverse(array $nodes);
+}
+```
+
+The `beforeTraverse()` and `afterTraverse()` methods are called before and after the traversal
+respectively, and are passed the entire AST. They can be used to perform any necessary state
+setup or cleanup.
+
+The `enterNode()` method is called when a node is first encountered, before its children are
+processed ("preorder"). The `leaveNode()` method is called after all children have been visited
+("postorder").
+
+For example, if we have the following excerpt of an AST
+
+```
+Expr_FuncCall(
+    name: Name(
+        parts: array(
+            0: printLine
+        )
+    )
+    args: array(
+        0: Arg(
+            value: Scalar_String(
+                value: Hello World!!!
+            )
+            byRef: false
+            unpack: false
+        )
+    )
+)
+```
+
+then the enter/leave methods will be called in the following order:
+
+```
+enterNode(Expr_FuncCall)
+enterNode(Name)
+leaveNode(Name)
+enterNode(Arg)
+enterNode(Scalar_String)
+leaveNode(Scalar_String)
+leaveNode(Arg)
+leaveNode(Expr_FuncCall)
+```
+
+A common pattern is that `enterNode` is used to collect some information and then `leaveNode`
+performs modifications based on that. At the time when `leaveNode` is called, all the code inside
+the node will have already been visited and necessary information collected.
+
+As you usually do not want to implement all four methods, it is recommended that you extend
+`NodeVisitorAbstract` instead of implementing the interface directly. The abstract class provides
+empty default implementations.
+
+Modifying the AST
+-----------------
+
+There are a number of ways in which the AST can be modified from inside a node visitor. The first
+and simplest is to simply change AST properties inside the visitor:
+
+```php
+public function leaveNode(Node $node) {
+    if ($node instanceof Node\Scalar\LNumber) {
+        // increment all integer literals
+        $node->value++;
+    }
+}
+```
+
+The second is to replace a node entirely by returning a new node:
+
+```php
+public function leaveNode(Node $node) {
+    if ($node instanceof Node\Expr\BinaryOp\BooleanAnd) {
+        // Convert all $a && $b expressions into !($a && $b)
+        return new Node\Expr\BooleanNot($node);
+    }
+}
+```
+
+Doing this is supported both inside enterNode and leaveNode. However, you have to be mindful about
+where you perform the replacement: If a node is replaced in enterNode, then the recursive traversal
+will also consider the children of the new node. If you aren't careful, this can lead to infinite
+recursion. For example, let's take the previous code sample and use enterNode instead:
+
+```php
+public function enterNode(Node $node) {
+    if ($node instanceof Node\Expr\BinaryOp\BooleanAnd) {
+        // Convert all $a && $b expressions into !($a && $b)
+        return new Node\Expr\BooleanNot($node);
+    }
+}
+```
+
+Now `$a && $b` will be replaced by `!($a && $b)`. Then the traverser will go into the first (and
+only) child of `!($a && $b)`, which is `$a && $b`. The transformation applies again and we end up
+with `!!($a && $b)`. This will continue until PHP hits the memory limit.
+
+Finally, two special replacement types are supported only by leaveNode. The first is removal of a
+node:
+
+```php
+public function leaveNode(Node $node) {
+    if ($node instanceof Node\Stmt\Return_) {
+        // Remove all return statements
+        return NodeTraverser::REMOVE_NODE;
+    }
+}
+```
+
+Node removal only works if the parent structure is an array. This means that usually it only makes
+sense to remove nodes of type `Node\Stmt`, as they always occur inside statement lists (and a few
+more node types like `Arg` or `Expr\ArrayItem`, which are also always part of lists).
+
+On the other hand, removing a `Node\Expr` does not make sense: If you have `$a * $b`, there is no
+meaningful way in which the `$a` part could be removed. If you want to remove an expression, you
+generally want to remove it together with a surrounding expression statement:
+
+```php
+public function leaveNode(Node $node) {
+    if ($node instanceof Node\Stmt\Expression
+        && $node->expr instanceof Node\Expr\FuncCall
+        && $node->expr->name instanceof Node\Name
+        && $node->expr->name->toString() === 'var_dump'
+    ) {
+        return NodeTraverser::REMOVE_NODE;
+    }
+}
+```
+
+This example will remove all calls to `var_dump()` which occur as expression statements. This means
+that `var_dump($a);` will be removed, but `if (var_dump($a))` will not be removed (and there is no
+obvious way in which it can be removed).
+
+Next to removing nodes, it is also possible to replace one node with multiple nodes. Again, this
+only works inside leaveNode and only if the parent structure is an array.
+
+```php
+public function leaveNode(Node $node) {
+    if ($node instanceof Node\Stmt\Return_ && $node->expr !== null) {
+        // Convert "return foo();" into "$retval = foo(); return $retval;"
+        $var = new Node\Expr\Variable('retval');
+        return [
+            new Node\Stmt\Expression(new Node\Expr\Assign($var, $node->expr)),
+            new Node\Stmt\Return_($var),
+        ];
+    }
+}
+```
+
+Short-circuiting traversal
+--------------------------
+
+An AST can easily contain thousands of nodes, and traversing over all of them may be slow,
+especially if you have more than one visitor. In some cases, it is possible to avoid a full
+traversal.
+
+If you are looking for all class declarations in a file (and assuming you're not interested in
+anonymous classes), you know that once you've seen a class declaration, there is no point in also
+checking all it's child nodes, because PHP does not allow nesting classes. In this case, you can
+instruct the traverser to not recurse into the class node:
+
+```php
+private $classes = [];
+public function enterNode(Node $node) {
+    if ($node instanceof Node\Stmt\Class_) {
+        $this->classes[] = $node;
+        return NodeTraverser::DONT_TRAVERSE_CHILDREN;
+    }
+}
+```
+
+Of course, this option is only available in enterNode, because it's already too late by the time
+leaveNode is reached.
+
+If you are only looking for one specific node, it is also possible to abort the traversal entirely
+after finding it. For example, if you are looking for the node of a class with a certain name (and
+discounting exotic cases like conditionally defining a class two times), you can stop traversal
+once you found it:
+
+```php
+private $class = null;
+public function enterNode(Node $node) {
+    if ($node instanceof Node\Stmt\Class_ &&
+        $node->namespacedName->toString() === 'Foo\Bar\Baz'
+    ) {
+        $this->class = $node;
+        return NodeTraverser::STOP_TRAVERSAL;
+    }
+}
+```
+
+This works both in enterNode and leaveNode. Note that this particular case can also be more easily
+handled using a NodeFinder, which will be introduced below.
+
+Multiple visitors
+-----------------
+
+A single traverser can be used with multiple visitors:
+
+```php
+$traverser = new NodeTraverser;
+$traverser->addVisitor($visitorA);
+$traverser->addVisitor($visitorB);
+$stmts = $traverser->traverse($stmts);
+```
+
+It is important to understand that if a traverser is run with multiple visitors, the visitors will
+be interleaved. Given the following AST excerpt
+
+```
+Stmt_Return(
+    expr: Expr_Variable(
+        name: foobar
+    )
+)
+```
+
+the following method calls will be performed:
+
+```php
+$visitorA->enterNode(Stmt_Return)
+$visitorB->enterNode(Stmt_Return)
+$visitorA->enterNode(Expr_Variable)
+$visitorB->enterNode(Expr_Variable)
+$visitorA->leaveNode(Expr_Variable)
+$visitorB->leaveNode(Expr_Variable)
+$visitorA->leaveNode(Stmt_Return)
+$visitorB->leaveNode(Stmt_Return)
+```
+
+That is, when visiting a node, enterNode and leaveNode will always be called for all visitors.
+Running multiple visitors in parallel improves performance, as the AST only has to be traversed
+once. However, it is not always possible to write visitors in a way that allows interleaved
+execution. In this case, you can always fall back to performing multiple traversals:
+
+```php
+$traverserA = new NodeTraverser;
+$traverserA->addVisitor($visitorA);
+$traverserB = new NodeTraverser;
+$traverserB->addVisitor($visitorB);
+$stmts = $traverserA->traverser($stmts);
+$stmts = $traverserB->traverser($stmts);
+```
+
+When using multiple visitors, it is important to understand how they interact with the various
+special enterNode/leaveNode return values:
+
+ * If *any* visitor returns `DONT_TRAVERSE_CHILDREN`, the children will be skipped for *all*
+   visitors.
+ * If *any* visitor returns `DONT_TRAVERSE_CURRENT_AND_CHILDREN`, the children will be skipped for *all*
+   visitors, and all *subsequent* visitors will not visit the current node.
+ * If *any* visitor returns `STOP_TRAVERSAL`, traversal is stopped for *all* visitors.
+ * If a visitor returns a replacement node, subsequent visitors will be passed the replacement node,
+   not the original one.
+ * If a visitor returns `REMOVE_NODE`, subsequent visitors will not see this node.
+ * If a visitor returns an array of replacement nodes, subsequent visitors will see neither the node
+   that was replaced, nor the replacement nodes.
+
+Simple node finding
+-------------------
+
+While the node visitor mechanism is very flexible, creating a node visitor can be overly cumbersome
+for minor tasks. For this reason a `NodeFinder` is provided, which can find AST nodes that either
+satisfy a certain callback, or which are instances of a certain node type. A couple of examples are
+shown in the following:
+
+```php
+use PhpParser\{Node, NodeFinder};
+
+$nodeFinder = new NodeFinder;
+
+// Find all class nodes.
+$classes = $nodeFinder->findInstanceOf($stmts, Node\Stmt\Class_::class);
+
+// Find all classes that extend another class
+$extendingClasses = $nodeFinder->find($stmts, function(Node $node) {
+    return $node instanceof Node\Stmt\Class_
+        && $node->extends !== null;
+});
+
+// Find first class occurring in the AST. Returns null if no class exists.
+$class = $nodeFinder->findFirstInstanceOf($stmts, Node\Stmt\Class_::class);
+
+// Find first class that has name $name
+$class = $nodeFinder->findFirst($stmts, function(Node $node) use ($name) {
+    return $node instanceof Node\Stmt\Class_
+        && $node->resolvedName->toString() === $name;
+});
+```
+
+Internally, the `NodeFinder` also uses a node traverser. It only simplifies the interface for a
+common use case.
+
+Parent and sibling references
+-----------------------------
+
+The node visitor mechanism is somewhat rigid, in that it prescribes an order in which nodes should
+be accessed: From parents to children. However, it can often be convenient to operate in the
+reverse direction: When working on a node, you might want to check if the parent node satisfies a
+certain property.
+
+PHP-Parser does not add parent (or sibling) references to nodes by itself, but you can easily
+emulate this with a visitor. See the [FAQ](FAQ.markdown) for more information.

grammar/README.md

@@ -11,7 +11,7 @@ What do all those files mean?
 .phpy pseudo language
 =====================
 
-The `.y` file is a normal grammer in `kmyacc` (`yacc`) style, with some transformations
+The `.y` file is a normal grammar in `kmyacc` (`yacc`) style, with some transformations
 applied to it:
 
  * Nodes are created using the syntax `Name[..., ...]`. This is transformed into
@@ -21,8 +21,10 @@ applied to it:
 Building the parser
 ===================
 
-In order to rebuild the parser, you need [moriyoshi's fork of kmyacc](https://github.com/moriyoshi/kmyacc-forked).
-After you compiled/installed it, run the `rebuildParsers.php` script.
+Run `php grammar/rebuildParsers.php` to rebuild the parsers. Additional options:
 
-By default only the `Parser.php` is built. If you want to additionally emit debug symbols and create `y.output`, run the
-script with `--debug`. If you want to retain the preprocessed grammar pass `--keep-tmp-grammar`.
+ * The `KMYACC` environment variable can be used to specify an alternative `kmyacc` binary.
+   By default the `phpyacc` dev dependency will be used. To use the original `kmyacc`, you
+   need to compile [moriyoshi's fork](https://github.com/moriyoshi/kmyacc-forked).
+ * The `--debug` option enables emission of debug symbols and creates the `y.output` file.
+ * The `--keep-tmp-grammar` option preserves the preprocessed grammar file.

grammar/parser.template

@@ -32,8 +32,8 @@ class #(-p) extends \PhpParser\ParserAbstract
     protected $defaultAction = #(YYDEFAULT);
     protected $unexpectedTokenRule = #(YYUNEXPECTED);
 
-    protected $YY2TBLSTATE  = #(YY2TBLSTATE);
-    protected $YYNLSTATES   = #(YYNLSTATES);
+    protected $YY2TBLSTATE = #(YY2TBLSTATE);
+    protected $numNonLeafStates = #(YYNLSTATES);
 
     protected $symbolToName = array(
         #listvar terminals

grammar/php5.y

@@ -16,10 +16,15 @@ top_statement_list_ex:
 
 top_statement_list:
       top_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
+ampersand:
+      T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
+    | T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG
+;
+
 reserved_non_modifiers:
       T_INCLUDE | T_INCLUDE_ONCE | T_EVAL | T_REQUIRE | T_REQUIRE_ONCE | T_LOGICAL_OR | T_LOGICAL_XOR | T_LOGICAL_AND
     | T_INSTANCEOF | T_NEW | T_CLONE | T_EXIT | T_IF | T_ELSEIF | T_ELSE | T_ENDIF | T_ECHO | T_DO | T_WHILE
@@ -27,7 +32,8 @@ reserved_non_modifiers:
     | T_FINALLY | T_THROW | T_USE | T_INSTEADOF | T_GLOBAL | T_VAR | T_UNSET | T_ISSET | T_EMPTY | T_CONTINUE | T_GOTO
     | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT
     | T_BREAK | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE | T_CLASS
-    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER
+    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER | T_FN
+    | T_MATCH
 ;
 
 semi_reserved:
@@ -44,13 +50,18 @@ identifier:
       T_STRING                                              { $$ = Node\Identifier[$1]; }
 ;
 
-namespace_name_parts:
-      T_STRING                                              { init($1); }
-    | namespace_name_parts T_NS_SEPARATOR T_STRING          { push($1, $3); }
+reserved_non_modifiers_identifier:
+      reserved_non_modifiers                                { $$ = Node\Identifier[$1]; }
 ;
 
 namespace_name:
-      namespace_name_parts                                  { $$ = Name[$1]; }
+      T_STRING                                              { $$ = Name[$1]; }
+    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
+;
+
+legacy_namespace_name:
+      namespace_name                                        { $$ = $1; }
+    | T_NAME_FULLY_QUALIFIED                                { $$ = Name[substr($1, 1)]; }
 ;
 
 plain_variable:
@@ -86,16 +97,11 @@ use_type:
     | T_CONST                                               { $$ = Stmt\Use_::TYPE_CONSTANT; }
 ;
 
-/* Using namespace_name_parts here to avoid s/r conflict on T_NS_SEPARATOR */
 group_use_declaration:
-      T_USE use_type namespace_name_parts T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
-          { $$ = Stmt\GroupUse[new Name($3, stackAttributes(#3)), $6, $2]; }
-    | T_USE use_type T_NS_SEPARATOR namespace_name_parts T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
-          { $$ = Stmt\GroupUse[new Name($4, stackAttributes(#4)), $7, $2]; }
-    | T_USE namespace_name_parts T_NS_SEPARATOR '{' inline_use_declarations '}'
-          { $$ = Stmt\GroupUse[new Name($2, stackAttributes(#2)), $5, Stmt\Use_::TYPE_UNKNOWN]; }
-    | T_USE T_NS_SEPARATOR namespace_name_parts T_NS_SEPARATOR '{' inline_use_declarations '}'
-          { $$ = Stmt\GroupUse[new Name($3, stackAttributes(#3)), $6, Stmt\Use_::TYPE_UNKNOWN]; }
+      T_USE use_type legacy_namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
+          { $$ = Stmt\GroupUse[$3, $6, $2]; }
+    | T_USE legacy_namespace_name T_NS_SEPARATOR '{' inline_use_declarations '}'
+          { $$ = Stmt\GroupUse[$2, $5, Stmt\Use_::TYPE_UNKNOWN]; }
 ;
 
 unprefixed_use_declarations:
@@ -122,8 +128,10 @@ unprefixed_use_declaration:
 ;
 
 use_declaration:
-      unprefixed_use_declaration                            { $$ = $1; }
-    | T_NS_SEPARATOR unprefixed_use_declaration             { $$ = $2; }
+      legacy_namespace_name
+          { $$ = Stmt\UseUse[$1, null, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #1); }
+    | legacy_namespace_name T_AS identifier
+          { $$ = Stmt\UseUse[$1, $3, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #3); }
 ;
 
 inline_use_declaration:
@@ -156,7 +164,7 @@ inner_statement_list_ex:
 
 inner_statement_list:
       inner_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -243,7 +251,12 @@ variables_list:
 
 optional_ref:
       /* empty */                                           { $$ = false; }
-    | '&'                                                   { $$ = true; }
+    | ampersand                                             { $$ = true; }
+;
+
+optional_arg_ref:
+      /* empty */                                           { $$ = false; }
+    | T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG                 { $$ = true; }
 ;
 
 optional_ellipsis:
@@ -251,8 +264,13 @@ optional_ellipsis:
     | T_ELLIPSIS                                            { $$ = true; }
 ;
 
+identifier_maybe_readonly:
+      identifier                                            { $$ = $1; }
+    | T_READONLY                                            { $$ = Node\Identifier[$1]; }
+;
+
 function_declaration_statement:
-    T_FUNCTION optional_ref identifier '(' parameter_list ')' optional_return_type '{' inner_statement_list '}'
+    T_FUNCTION optional_ref identifier_maybe_readonly '(' parameter_list ')' optional_return_type '{' inner_statement_list '}'
         { $$ = Stmt\Function_[$3, ['byRef' => $2, 'params' => $5, 'returnType' => $7, 'stmts' => $9]]; }
 ;
 
@@ -331,8 +349,8 @@ case_list:
 ;
 
 case:
-      T_CASE expr case_separator inner_statement_list       { $$ = Stmt\Case_[$2, $4]; }
-    | T_DEFAULT case_separator inner_statement_list         { $$ = Stmt\Case_[null, $3]; }
+      T_CASE expr case_separator inner_statement_list_ex    { $$ = Stmt\Case_[$2, $4]; }
+    | T_DEFAULT case_separator inner_statement_list_ex      { $$ = Stmt\Case_[null, $3]; }
 ;
 
 case_separator:
@@ -375,7 +393,7 @@ new_else_single:
 
 foreach_variable:
       variable                                              { $$ = array($1, false); }
-    | '&' variable                                          { $$ = array($2, true); }
+    | ampersand variable                                    { $$ = array($2, true); }
     | list_expr                                             { $$ = array($1, false); }
 ;
 
@@ -390,9 +408,9 @@ non_empty_parameter_list:
 ;
 
 parameter:
-      optional_param_type optional_ref optional_ellipsis plain_variable
+      optional_param_type optional_arg_ref optional_ellipsis plain_variable
           { $$ = Node\Param[$4, null, $1, $2, $3]; $this->checkParam($$); }
-    | optional_param_type optional_ref optional_ellipsis plain_variable '=' static_scalar
+    | optional_param_type optional_arg_ref optional_ellipsis plain_variable '=' static_scalar
           { $$ = Node\Param[$4, $6, $1, $2, $3]; $this->checkParam($$); }
 ;
 
@@ -425,7 +443,7 @@ non_empty_argument_list:
 
 argument:
       expr                                                  { $$ = Node\Arg[$1, false, false]; }
-    | '&' variable                                          { $$ = Node\Arg[$2, true, false]; }
+    | ampersand variable                                    { $$ = Node\Arg[$2, true, false]; }
     | T_ELLIPSIS expr                                       { $$ = Node\Arg[$2, false, true]; }
 ;
 
@@ -450,11 +468,17 @@ static_var:
     | plain_variable '=' static_scalar                      { $$ = Stmt\StaticVar[$1, $3]; }
 ;
 
-class_statement_list:
-      class_statement_list class_statement                  { push($1, $2); }
+class_statement_list_ex:
+      class_statement_list_ex class_statement               { if ($2 !== null) { push($1, $2); } else { $$ = $1; } }
     | /* empty */                                           { init(); }
 ;
 
+class_statement_list:
+      class_statement_list_ex
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
+            if ($nop !== null) { $1[] = $nop; } $$ = $1; }
+;
+
 class_statement:
       variable_modifiers property_declaration_list ';'
           { $$ = Stmt\Property[$1, $2]; $this->checkProperty($$, #1); }
@@ -484,7 +508,7 @@ trait_adaptation:
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], $3, null]; }
     | trait_method_reference T_AS identifier ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], null, $3]; }
-    | trait_method_reference T_AS reserved_non_modifiers ';'
+    | trait_method_reference T_AS reserved_non_modifiers_identifier ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], null, $3]; }
 ;
 
@@ -553,8 +577,8 @@ expr:
       variable                                              { $$ = $1; }
     | list_expr '=' expr                                    { $$ = Expr\Assign[$1, $3]; }
     | variable '=' expr                                     { $$ = Expr\Assign[$1, $3]; }
-    | variable '=' '&' variable                             { $$ = Expr\AssignRef[$1, $4]; }
-    | variable '=' '&' new_expr                             { $$ = Expr\AssignRef[$1, $4]; }
+    | variable '=' ampersand variable                       { $$ = Expr\AssignRef[$1, $4]; }
+    | variable '=' ampersand new_expr                       { $$ = Expr\AssignRef[$1, $4]; }
     | new_expr                                              { $$ = $1; }
     | T_CLONE expr                                          { $$ = Expr\Clone_[$2]; }
     | variable T_PLUS_EQUAL expr                            { $$ = Expr\AssignOp\Plus      [$1, $3]; }
@@ -569,6 +593,7 @@ expr:
     | variable T_SL_EQUAL expr                              { $$ = Expr\AssignOp\ShiftLeft [$1, $3]; }
     | variable T_SR_EQUAL expr                              { $$ = Expr\AssignOp\ShiftRight[$1, $3]; }
     | variable T_POW_EQUAL expr                             { $$ = Expr\AssignOp\Pow       [$1, $3]; }
+    | variable T_COALESCE_EQUAL expr                        { $$ = Expr\AssignOp\Coalesce  [$1, $3]; }
     | variable T_INC                                        { $$ = Expr\PostInc[$1]; }
     | T_INC variable                                        { $$ = Expr\PreInc [$2]; }
     | variable T_DEC                                        { $$ = Expr\PostDec[$1]; }
@@ -579,7 +604,8 @@ expr:
     | expr T_LOGICAL_AND expr                               { $$ = Expr\BinaryOp\LogicalAnd[$1, $3]; }
     | expr T_LOGICAL_XOR expr                               { $$ = Expr\BinaryOp\LogicalXor[$1, $3]; }
     | expr '|' expr                                         { $$ = Expr\BinaryOp\BitwiseOr [$1, $3]; }
-    | expr '&' expr                                         { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | expr T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG expr   { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | expr T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG expr       { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
     | expr '^' expr                                         { $$ = Expr\BinaryOp\BitwiseXor[$1, $3]; }
     | expr '.' expr                                         { $$ = Expr\BinaryOp\Concat    [$1, $3]; }
     | expr '+' expr                                         { $$ = Expr\BinaryOp\Plus      [$1, $3]; }
@@ -618,7 +644,10 @@ expr:
     | T_REQUIRE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE]; }
     | T_REQUIRE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE_ONCE]; }
     | T_INT_CAST expr                                       { $$ = Expr\Cast\Int_    [$2]; }
-    | T_DOUBLE_CAST expr                                    { $$ = Expr\Cast\Double  [$2]; }
+    | T_DOUBLE_CAST expr
+          { $attrs = attributes();
+            $attrs['kind'] = $this->getFloatCastKind($1);
+            $$ = new Expr\Cast\Double($2, $attrs); }
     | T_STRING_CAST expr                                    { $$ = Expr\Cast\String_ [$2]; }
     | T_ARRAY_CAST expr                                     { $$ = Expr\Cast\Array_  [$2]; }
     | T_OBJECT_CAST expr                                    { $$ = Expr\Cast\Object_ [$2]; }
@@ -665,9 +694,7 @@ array_expr:
 
 scalar_dereference:
       array_expr '[' dim_offset ']'                         { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | T_CONSTANT_ENCAPSED_STRING '[' dim_offset ']'
-          { $attrs = attributes(); $attrs['kind'] = strKind($1);
-            $$ = Expr\ArrayDimFetch[new Scalar\String_(Scalar\String_::parse($1), $attrs), $3]; }
+    | T_CONSTANT_ENCAPSED_STRING '[' dim_offset ']'         { $$ = Expr\ArrayDimFetch[Scalar\String_::fromString($1, attributes()), $3]; }
     | constant '[' dim_offset ']'                           { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | scalar_dereference '[' dim_offset ']'                 { $$ = Expr\ArrayDimFetch[$1, $3]; }
     /* alternative array syntax missing intentionally */
@@ -699,8 +726,13 @@ lexical_var:
       optional_ref plain_variable                           { $$ = Expr\ClosureUse[$2, $1]; }
 ;
 
+name_readonly:
+      T_READONLY                                            { $$ = Name[$1]; }
+;
+
 function_call:
       name argument_list                                    { $$ = Expr\FuncCall[$1, $2]; }
+    | name_readonly argument_list                           { $$ = Expr\FuncCall[$1, $2]; }
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM identifier_ex argument_list
           { $$ = Expr\StaticCall[$1, $3, $4]; }
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM '{' expr '}' argument_list
@@ -719,9 +751,10 @@ class_name:
 ;
 
 name:
-      namespace_name_parts                                  { $$ = Name[$1]; }
-    | T_NS_SEPARATOR namespace_name_parts                   { $$ = Name\FullyQualified[$2]; }
-    | T_NAMESPACE T_NS_SEPARATOR namespace_name_parts       { $$ = Name\Relative[$3]; }
+      T_STRING                                              { $$ = Name[$1]; }
+    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
+    | T_NAME_FULLY_QUALIFIED                                { $$ = Name\FullyQualified[substr($1, 1)]; }
+    | T_NAME_RELATIVE                                       { $$ = Name\Relative[substr($1, 10)]; }
 ;
 
 class_name_reference:
@@ -768,10 +801,8 @@ ctor_arguments:
 
 common_scalar:
       T_LNUMBER                                             { $$ = $this->parseLNumber($1, attributes(), true); }
-    | T_DNUMBER                                             { $$ = Scalar\DNumber[Scalar\DNumber::parse($1)]; }
-    | T_CONSTANT_ENCAPSED_STRING
-          { $attrs = attributes(); $attrs['kind'] = strKind($1);
-            $$ = new Scalar\String_(Scalar\String_::parse($1, false), $attrs); }
+    | T_DNUMBER                                             { $$ = Scalar\DNumber::fromString($1, attributes()); }
+    | T_CONSTANT_ENCAPSED_STRING                            { $$ = Scalar\String_::fromString($1, attributes(), false); }
     | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }
     | T_FILE                                                { $$ = Scalar\MagicConst\File[]; }
     | T_DIR                                                 { $$ = Scalar\MagicConst\Dir[]; }
@@ -781,11 +812,9 @@ common_scalar:
     | T_FUNC_C                                              { $$ = Scalar\MagicConst\Function_[]; }
     | T_NS_C                                                { $$ = Scalar\MagicConst\Namespace_[]; }
     | T_START_HEREDOC T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC
-          { $attrs = attributes(); setDocStringAttrs($attrs, $1);
-            $$ = new Scalar\String_(Scalar\String_::parseDocString($1, $2, false), $attrs); }
+          { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), false); }
     | T_START_HEREDOC T_END_HEREDOC
-          { $attrs = attributes(); setDocStringAttrs($attrs, $1);
-            $$ = new Scalar\String_('', $attrs); }
+          { $$ = $this->parseDocString($1, '', $2, attributes(), stackAttributes(#2), false); }
 ;
 
 static_scalar:
@@ -804,7 +833,10 @@ static_operation:
     | static_scalar T_LOGICAL_AND static_scalar             { $$ = Expr\BinaryOp\LogicalAnd[$1, $3]; }
     | static_scalar T_LOGICAL_XOR static_scalar             { $$ = Expr\BinaryOp\LogicalXor[$1, $3]; }
     | static_scalar '|' static_scalar                       { $$ = Expr\BinaryOp\BitwiseOr [$1, $3]; }
-    | static_scalar '&' static_scalar                       { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | static_scalar T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG static_scalar
+          { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | static_scalar T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG static_scalar
+          { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
     | static_scalar '^' static_scalar                       { $$ = Expr\BinaryOp\BitwiseXor[$1, $3]; }
     | static_scalar '.' static_scalar                       { $$ = Expr\BinaryOp\Concat    [$1, $3]; }
     | static_scalar '+' static_scalar                       { $$ = Expr\BinaryOp\Plus      [$1, $3]; }
@@ -846,8 +878,7 @@ scalar:
           { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
             parseEncapsed($2, '"', true); $$ = new Scalar\Encapsed($2, $attrs); }
     | T_START_HEREDOC encaps_list T_END_HEREDOC
-          { $attrs = attributes(); setDocStringAttrs($attrs, $1);
-            parseEncapsedDoc($2, true); $$ = new Scalar\Encapsed($2, $attrs); }
+          { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), true); }
 ;
 
 static_array_pair_list:
@@ -975,8 +1006,9 @@ non_empty_array_pair_list:
 array_pair:
       expr T_DOUBLE_ARROW expr                              { $$ = Expr\ArrayItem[$3, $1,   false]; }
     | expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
-    | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
-    | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
+    | expr T_DOUBLE_ARROW ampersand variable                { $$ = Expr\ArrayItem[$4, $1,   true]; }
+    | ampersand variable                                    { $$ = Expr\ArrayItem[$2, null, true]; }
+    | T_ELLIPSIS expr                                       { $$ = new Expr\ArrayItem($2, null, false, attributes(), true); }
 ;
 
 encaps_list:

grammar/php7.y

@@ -16,10 +16,15 @@ top_statement_list_ex:
 
 top_statement_list:
       top_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
+ampersand:
+      T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
+    | T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG
+;
+
 reserved_non_modifiers:
       T_INCLUDE | T_INCLUDE_ONCE | T_EVAL | T_REQUIRE | T_REQUIRE_ONCE | T_LOGICAL_OR | T_LOGICAL_XOR | T_LOGICAL_AND
     | T_INSTANCEOF | T_NEW | T_CLONE | T_EXIT | T_IF | T_ELSEIF | T_ELSE | T_ENDIF | T_ECHO | T_DO | T_WHILE
@@ -27,30 +32,42 @@ reserved_non_modifiers:
     | T_FINALLY | T_THROW | T_USE | T_INSTEADOF | T_GLOBAL | T_VAR | T_UNSET | T_ISSET | T_EMPTY | T_CONTINUE | T_GOTO
     | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT
     | T_BREAK | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE | T_CLASS
-    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER
+    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER | T_FN
+    | T_MATCH | T_ENUM
 ;
 
 semi_reserved:
       reserved_non_modifiers
-    | T_STATIC | T_ABSTRACT | T_FINAL | T_PRIVATE | T_PROTECTED | T_PUBLIC
+    | T_STATIC | T_ABSTRACT | T_FINAL | T_PRIVATE | T_PROTECTED | T_PUBLIC | T_READONLY
 ;
 
-identifier_ex:
+identifier_maybe_reserved:
       T_STRING                                              { $$ = Node\Identifier[$1]; }
     | semi_reserved                                         { $$ = Node\Identifier[$1]; }
 ;
 
-identifier:
+identifier_not_reserved:
       T_STRING                                              { $$ = Node\Identifier[$1]; }
 ;
 
-namespace_name_parts:
-      T_STRING                                              { init($1); }
-    | namespace_name_parts T_NS_SEPARATOR T_STRING          { push($1, $3); }
+reserved_non_modifiers_identifier:
+      reserved_non_modifiers                                { $$ = Node\Identifier[$1]; }
+;
+
+namespace_declaration_name:
+      T_STRING                                              { $$ = Name[$1]; }
+    | semi_reserved                                         { $$ = Name[$1]; }
+    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
 ;
 
 namespace_name:
-      namespace_name_parts                                  { $$ = Name[$1]; }
+      T_STRING                                              { $$ = Name[$1]; }
+    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
+;
+
+legacy_namespace_name:
+      namespace_name                                        { $$ = $1; }
+    | T_NAME_FULLY_QUALIFIED                                { $$ = Name[substr($1, 1)]; }
 ;
 
 plain_variable:
@@ -70,6 +87,31 @@ no_comma:
 optional_comma:
       /* empty */
     | ','
+;
+
+attribute_decl:
+      class_name                                            { $$ = Node\Attribute[$1, []]; }
+    | class_name argument_list                              { $$ = Node\Attribute[$1, $2]; }
+;
+
+attribute_group:
+      attribute_decl                                        { init($1); }
+    | attribute_group ',' attribute_decl                    { push($1, $3); }
+;
+
+attribute:
+      T_ATTRIBUTE attribute_group optional_comma ']'        { $$ = Node\AttributeGroup[$2]; }
+;
+
+attributes:
+      attribute                                             { init($1); }
+    | attributes attribute                                  { push($1, $2); }
+;
+
+optional_attributes:
+      /* empty */                                           { $$ = []; }
+    | attributes                                            { $$ = $1; }
+;
 
 top_statement:
       statement                                             { $$ = $1; }
@@ -77,11 +119,11 @@ top_statement:
     | class_declaration_statement                           { $$ = $1; }
     | T_HALT_COMPILER
           { $$ = Stmt\HaltCompiler[$this->lexer->handleHaltCompiler()]; }
-    | T_NAMESPACE namespace_name semi
+    | T_NAMESPACE namespace_declaration_name semi
           { $$ = Stmt\Namespace_[$2, null];
             $$->setAttribute('kind', Stmt\Namespace_::KIND_SEMICOLON);
             $this->checkNamespace($$); }
-    | T_NAMESPACE namespace_name '{' top_statement_list '}'
+    | T_NAMESPACE namespace_declaration_name '{' top_statement_list '}'
           { $$ = Stmt\Namespace_[$2, $4];
             $$->setAttribute('kind', Stmt\Namespace_::KIND_BRACED);
             $this->checkNamespace($$); }
@@ -100,16 +142,11 @@ use_type:
     | T_CONST                                               { $$ = Stmt\Use_::TYPE_CONSTANT; }
 ;
 
-/* Using namespace_name_parts here to avoid s/r conflict on T_NS_SEPARATOR */
 group_use_declaration:
-      T_USE use_type namespace_name_parts T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
-          { $$ = Stmt\GroupUse[new Name($3, stackAttributes(#3)), $6, $2]; }
-    | T_USE use_type T_NS_SEPARATOR namespace_name_parts T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
-          { $$ = Stmt\GroupUse[new Name($4, stackAttributes(#4)), $7, $2]; }
-    | T_USE namespace_name_parts T_NS_SEPARATOR '{' inline_use_declarations '}'
-          { $$ = Stmt\GroupUse[new Name($2, stackAttributes(#2)), $5, Stmt\Use_::TYPE_UNKNOWN]; }
-    | T_USE T_NS_SEPARATOR namespace_name_parts T_NS_SEPARATOR '{' inline_use_declarations '}'
-          { $$ = Stmt\GroupUse[new Name($3, stackAttributes(#3)), $6, Stmt\Use_::TYPE_UNKNOWN]; }
+      T_USE use_type legacy_namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
+          { $$ = Stmt\GroupUse[$3, $6, $2]; }
+    | T_USE legacy_namespace_name T_NS_SEPARATOR '{' inline_use_declarations '}'
+          { $$ = Stmt\GroupUse[$2, $5, Stmt\Use_::TYPE_UNKNOWN]; }
 ;
 
 unprefixed_use_declarations:
@@ -144,13 +181,15 @@ non_empty_inline_use_declarations:
 unprefixed_use_declaration:
       namespace_name
           { $$ = Stmt\UseUse[$1, null, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #1); }
-    | namespace_name T_AS identifier
+    | namespace_name T_AS identifier_not_reserved
           { $$ = Stmt\UseUse[$1, $3, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #3); }
 ;
 
 use_declaration:
-      unprefixed_use_declaration                            { $$ = $1; }
-    | T_NS_SEPARATOR unprefixed_use_declaration             { $$ = $2; }
+      legacy_namespace_name
+          { $$ = Stmt\UseUse[$1, null, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #1); }
+    | legacy_namespace_name T_AS identifier_not_reserved
+          { $$ = Stmt\UseUse[$1, $3, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #3); }
 ;
 
 inline_use_declaration:
@@ -169,7 +208,7 @@ non_empty_constant_declaration_list:
 ;
 
 constant_declaration:
-    identifier '=' expr                                     { $$ = Node\Const_[$1, $3]; }
+    identifier_not_reserved '=' expr                        { $$ = Node\Const_[$1, $3]; }
 ;
 
 class_const_list:
@@ -182,7 +221,10 @@ non_empty_class_const_list:
 ;
 
 class_const:
-    identifier_ex '=' expr                                  { $$ = Node\Const_[$1, $3]; }
+      T_STRING '=' expr
+          { $$ = Node\Const_[new Node\Identifier($1, stackAttributes(#1)), $3]; }
+    | semi_reserved '=' expr
+          { $$ = Node\Const_[new Node\Identifier($1, stackAttributes(#1)), $3]; }
 ;
 
 inner_statement_list_ex:
@@ -192,7 +234,7 @@ inner_statement_list_ex:
 
 inner_statement_list:
       inner_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -228,20 +270,30 @@ non_empty_statement:
     | T_RETURN optional_expr semi                           { $$ = Stmt\Return_[$2]; }
     | T_GLOBAL global_var_list semi                         { $$ = Stmt\Global_[$2]; }
     | T_STATIC static_var_list semi                         { $$ = Stmt\Static_[$2]; }
-    | T_ECHO expr_list semi                                 { $$ = Stmt\Echo_[$2]; }
+    | T_ECHO expr_list_forbid_comma semi                    { $$ = Stmt\Echo_[$2]; }
     | T_INLINE_HTML                                         { $$ = Stmt\InlineHTML[$1]; }
-    | expr semi                                             { $$ = Stmt\Expression[$1]; }
+    | expr semi {
+        $e = $1;
+        if ($e instanceof Expr\Throw_) {
+            // For backwards-compatibility reasons, convert throw in statement position into
+            // Stmt\Throw_ rather than Stmt\Expression(Expr\Throw_).
+            $$ = Stmt\Throw_[$e->expr];
+        } else {
+            $$ = Stmt\Expression[$e];
+        }
+    }
     | T_UNSET '(' variables_list ')' semi                   { $$ = Stmt\Unset_[$3]; }
     | T_FOREACH '(' expr T_AS foreach_variable ')' foreach_statement
           { $$ = Stmt\Foreach_[$3, $5[0], ['keyVar' => null, 'byRef' => $5[1], 'stmts' => $7]]; }
     | T_FOREACH '(' expr T_AS variable T_DOUBLE_ARROW foreach_variable ')' foreach_statement
           { $$ = Stmt\Foreach_[$3, $7[0], ['keyVar' => $5, 'byRef' => $7[1], 'stmts' => $9]]; }
+    | T_FOREACH '(' expr error ')' foreach_statement
+          { $$ = Stmt\Foreach_[$3, new Expr\Error(stackAttributes(#4)), ['stmts' => $6]]; }
     | T_DECLARE '(' declare_list ')' declare_statement      { $$ = Stmt\Declare_[$3, $5]; }
     | T_TRY '{' inner_statement_list '}' catches optional_finally
           { $$ = Stmt\TryCatch[$3, $5, $6]; $this->checkTryCatch($$); }
-    | T_THROW expr semi                                     { $$ = Stmt\Throw_[$2]; }
-    | T_GOTO identifier semi                                { $$ = Stmt\Goto_[$2]; }
-    | identifier ':'                                        { $$ = Stmt\Label[$1]; }
+    | T_GOTO identifier_not_reserved semi                   { $$ = Stmt\Goto_[$2]; }
+    | identifier_not_reserved ':'                           { $$ = Stmt\Label[$1]; }
     | error                                                 { $$ = array(); /* means: no statement */ }
 ;
 
@@ -263,7 +315,7 @@ name_union:
 ;
 
 catch:
-    T_CATCH '(' name_union plain_variable ')' '{' inner_statement_list '}'
+    T_CATCH '(' name_union optional_plain_variable ')' '{' inner_statement_list '}'
         { $$ = Stmt\Catch_[$3, $4, $7]; }
 ;
 
@@ -273,7 +325,7 @@ optional_finally:
 ;
 
 variables_list:
-      non_empty_variables_list no_comma                     { $$ = $1; }
+      non_empty_variables_list optional_comma               { $$ = $1; }
 ;
 
 non_empty_variables_list:
@@ -283,7 +335,12 @@ non_empty_variables_list:
 
 optional_ref:
       /* empty */                                           { $$ = false; }
-    | '&'                                                   { $$ = true; }
+    | ampersand                                             { $$ = true; }
+;
+
+optional_arg_ref:
+      /* empty */                                           { $$ = false; }
+    | T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG                 { $$ = true; }
 ;
 
 optional_ellipsis:
@@ -291,26 +348,63 @@ optional_ellipsis:
     | T_ELLIPSIS                                            { $$ = true; }
 ;
 
+block_or_error:
+      '{' inner_statement_list '}'                          { $$ = $2; }
+    | error                                                 { $$ = []; }
+;
+
+identifier_maybe_readonly:
+      identifier_not_reserved                               { $$ = $1; }
+    | T_READONLY                                            { $$ = Node\Identifier[$1]; }
+;
+
 function_declaration_statement:
-    T_FUNCTION optional_ref identifier '(' parameter_list ')' optional_return_type '{' inner_statement_list '}'
-        { $$ = Stmt\Function_[$3, ['byRef' => $2, 'params' => $5, 'returnType' => $7, 'stmts' => $9]]; }
+      T_FUNCTION optional_ref identifier_maybe_readonly '(' parameter_list ')' optional_return_type block_or_error
+          { $$ = Stmt\Function_[$3, ['byRef' => $2, 'params' => $5, 'returnType' => $7, 'stmts' => $8, 'attrGroups' => []]]; }
+    | attributes T_FUNCTION optional_ref identifier_maybe_readonly '(' parameter_list ')' optional_return_type block_or_error
+          { $$ = Stmt\Function_[$4, ['byRef' => $3, 'params' => $6, 'returnType' => $8, 'stmts' => $9, 'attrGroups' => $1]]; }
 ;
 
 class_declaration_statement:
-      class_entry_type identifier extends_from implements_list '{' class_statement_list '}'
-          { $$ = Stmt\Class_[$2, ['type' => $1, 'extends' => $3, 'implements' => $4, 'stmts' => $6]];
+      class_entry_type identifier_not_reserved extends_from implements_list '{' class_statement_list '}'
+          { $$ = Stmt\Class_[$2, ['type' => $1, 'extends' => $3, 'implements' => $4, 'stmts' => $6, 'attrGroups' => []]];
             $this->checkClass($$, #2); }
-    | T_INTERFACE identifier interface_extends_list '{' class_statement_list '}'
-          { $$ = Stmt\Interface_[$2, ['extends' => $3, 'stmts' => $5]];
-            $this->checkInterface($$, #2); }
-    | T_TRAIT identifier '{' class_statement_list '}'
-          { $$ = Stmt\Trait_[$2, ['stmts' => $4]]; }
+    | attributes class_entry_type identifier_not_reserved extends_from implements_list '{' class_statement_list '}'
+          { $$ = Stmt\Class_[$3, ['type' => $2, 'extends' => $4, 'implements' => $5, 'stmts' => $7, 'attrGroups' => $1]];
+            $this->checkClass($$, #3); }
+    | optional_attributes T_INTERFACE identifier_not_reserved interface_extends_list '{' class_statement_list '}'
+          { $$ = Stmt\Interface_[$3, ['extends' => $4, 'stmts' => $6, 'attrGroups' => $1]];
+            $this->checkInterface($$, #3); }
+    | optional_attributes T_TRAIT identifier_not_reserved '{' class_statement_list '}'
+          { $$ = Stmt\Trait_[$3, ['stmts' => $5, 'attrGroups' => $1]]; }
+    | optional_attributes T_ENUM identifier_not_reserved enum_scalar_type implements_list '{' class_statement_list '}'
+          { $$ = Stmt\Enum_[$3, ['scalarType' => $4, 'implements' => $5, 'stmts' => $7, 'attrGroups' => $1]];
+            $this->checkEnum($$, #3); }
+;
+
+enum_scalar_type:
+      /* empty */                                           { $$ = null; }
+    | ':' type                                              { $$ = $2; }
+
+enum_case_expr:
+      /* empty */                                           { $$ = null; }
+    | '=' expr                                              { $$ = $2; }
 ;
 
 class_entry_type:
       T_CLASS                                               { $$ = 0; }
-    | T_ABSTRACT T_CLASS                                    { $$ = Stmt\Class_::MODIFIER_ABSTRACT; }
-    | T_FINAL T_CLASS                                       { $$ = Stmt\Class_::MODIFIER_FINAL; }
+    | class_modifiers T_CLASS                               { $$ = $1; }
+;
+
+class_modifiers:
+      class_modifier                                        { $$ = $1; }
+    | class_modifiers class_modifier                        { $this->checkClassModifier($1, $2, #2); $$ = $1 | $2; }
+;
+
+class_modifier:
+      T_ABSTRACT                                            { $$ = Stmt\Class_::MODIFIER_ABSTRACT; }
+    | T_FINAL                                               { $$ = Stmt\Class_::MODIFIER_FINAL; }
+    | T_READONLY                                            { $$ = Stmt\Class_::MODIFIER_READONLY; }
 ;
 
 extends_from:
@@ -363,7 +457,7 @@ non_empty_declare_list:
 ;
 
 declare_list_element:
-      identifier '=' expr                                   { $$ = Stmt\DeclareDeclare[$1, $3]; }
+      identifier_not_reserved '=' expr                      { $$ = Stmt\DeclareDeclare[$1, $3]; }
 ;
 
 switch_case_list:
@@ -379,8 +473,8 @@ case_list:
 ;
 
 case:
-      T_CASE expr case_separator inner_statement_list       { $$ = Stmt\Case_[$2, $4]; }
-    | T_DEFAULT case_separator inner_statement_list         { $$ = Stmt\Case_[null, $3]; }
+      T_CASE expr case_separator inner_statement_list_ex    { $$ = Stmt\Case_[$2, $4]; }
+    | T_DEFAULT case_separator inner_statement_list_ex      { $$ = Stmt\Case_[null, $3]; }
 ;
 
 case_separator:
@@ -388,6 +482,25 @@ case_separator:
     | ';'
 ;
 
+match:
+      T_MATCH '(' expr ')' '{' match_arm_list '}'           { $$ = Expr\Match_[$3, $6]; }
+;
+
+match_arm_list:
+      /* empty */                                           { $$ = []; }
+    | non_empty_match_arm_list optional_comma               { $$ = $1; }
+;
+
+non_empty_match_arm_list:
+      match_arm                                             { init($1); }
+    | non_empty_match_arm_list ',' match_arm                { push($1, $3); }
+;
+
+match_arm:
+      expr_list_allow_comma T_DOUBLE_ARROW expr             { $$ = Node\MatchArm[$1, $3]; }
+    | T_DEFAULT optional_comma T_DOUBLE_ARROW expr          { $$ = Node\MatchArm[null, $4]; }
+;
+
 while_statement:
       statement                                             { $$ = toArray($1); }
     | ':' inner_statement_list T_ENDWHILE ';'               { $$ = $2; }
@@ -408,7 +521,8 @@ new_elseif_list:
 ;
 
 new_elseif:
-     T_ELSEIF '(' expr ')' ':' inner_statement_list         { $$ = Stmt\ElseIf_[$3, $6]; }
+     T_ELSEIF '(' expr ')' ':' inner_statement_list
+         { $$ = Stmt\ElseIf_[$3, $6]; $this->fixupAlternativeElse($$); }
 ;
 
 else_single:
@@ -418,18 +532,19 @@ else_single:
 
 new_else_single:
       /* empty */                                           { $$ = null; }
-    | T_ELSE ':' inner_statement_list                       { $$ = Stmt\Else_[$3]; }
+    | T_ELSE ':' inner_statement_list
+          { $$ = Stmt\Else_[$3]; $this->fixupAlternativeElse($$); }
 ;
 
 foreach_variable:
       variable                                              { $$ = array($1, false); }
-    | '&' variable                                          { $$ = array($2, true); }
+    | ampersand variable                                    { $$ = array($2, true); }
     | list_expr                                             { $$ = array($1, false); }
     | array_short_syntax                                    { $$ = array($1, false); }
 ;
 
 parameter_list:
-      non_empty_parameter_list no_comma                     { $$ = $1; }
+      non_empty_parameter_list optional_comma               { $$ = $1; }
     | /* empty */                                           { $$ = array(); }
 ;
 
@@ -438,37 +553,118 @@ non_empty_parameter_list:
     | non_empty_parameter_list ',' parameter                { push($1, $3); }
 ;
 
+optional_property_modifiers:
+      /* empty */               { $$ = 0; }
+    | optional_property_modifiers property_modifier
+          { $this->checkModifier($1, $2, #2); $$ = $1 | $2; }
+;
+
+property_modifier:
+      T_PUBLIC                  { $$ = Stmt\Class_::MODIFIER_PUBLIC; }
+    | T_PROTECTED               { $$ = Stmt\Class_::MODIFIER_PROTECTED; }
+    | T_PRIVATE                 { $$ = Stmt\Class_::MODIFIER_PRIVATE; }
+    | T_READONLY                { $$ = Stmt\Class_::MODIFIER_READONLY; }
+;
+
 parameter:
-      optional_param_type optional_ref optional_ellipsis plain_variable
-          { $$ = Node\Param[$4, null, $1, $2, $3]; $this->checkParam($$); }
-    | optional_param_type optional_ref optional_ellipsis plain_variable '=' expr
-          { $$ = Node\Param[$4, $6, $1, $2, $3]; $this->checkParam($$); }
+      optional_attributes optional_property_modifiers optional_type_without_static
+      optional_arg_ref optional_ellipsis plain_variable
+          { $$ = new Node\Param($6, null, $3, $4, $5, attributes(), $2, $1);
+            $this->checkParam($$); }
+    | optional_attributes optional_property_modifiers optional_type_without_static
+      optional_arg_ref optional_ellipsis plain_variable '=' expr
+          { $$ = new Node\Param($6, $8, $3, $4, $5, attributes(), $2, $1);
+            $this->checkParam($$); }
+    | optional_attributes optional_property_modifiers optional_type_without_static
+      optional_arg_ref optional_ellipsis error
+          { $$ = new Node\Param(Expr\Error[], null, $3, $4, $5, attributes(), $2, $1); }
 ;
 
 type_expr:
       type                                                  { $$ = $1; }
     | '?' type                                              { $$ = Node\NullableType[$2]; }
+    | union_type                                            { $$ = Node\UnionType[$1]; }
+    | intersection_type                                     { $$ = $1; }
 ;
 
 type:
+      type_without_static                                   { $$ = $1; }
+    | T_STATIC                                              { $$ = Node\Name['static']; }
+;
+
+type_without_static:
       name                                                  { $$ = $this->handleBuiltinTypes($1); }
     | T_ARRAY                                               { $$ = Node\Identifier['array']; }
     | T_CALLABLE                                            { $$ = Node\Identifier['callable']; }
 ;
 
-optional_param_type:
+union_type_element:
+                type { $$ = $1; }
+        |        '(' intersection_type ')' { $$ = $2; }
+;
+
+union_type:
+      union_type_element '|' union_type_element             { init($1, $3); }
+    | union_type '|' union_type_element                     { push($1, $3); }
+;
+
+union_type_without_static_element:
+                type_without_static { $$ = $1; }
+        |        '(' intersection_type_without_static ')' { $$ = $2; }
+;
+
+union_type_without_static:
+      union_type_without_static_element '|' union_type_without_static_element   { init($1, $3); }
+    | union_type_without_static '|' union_type_without_static_element           { push($1, $3); }
+;
+
+intersection_type_list:
+      type T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG type   { init($1, $3); }
+    | intersection_type_list T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG type
+          { push($1, $3); }
+;
+
+intersection_type:
+      intersection_type_list { $$ = Node\IntersectionType[$1]; }
+;
+
+intersection_type_without_static_list:
+      type_without_static T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG type_without_static
+          { init($1, $3); }
+    | intersection_type_without_static_list T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG type_without_static
+          { push($1, $3); }
+;
+
+intersection_type_without_static:
+      intersection_type_without_static_list { $$ = Node\IntersectionType[$1]; }
+;
+
+type_expr_without_static:
+      type_without_static                                   { $$ = $1; }
+    | '?' type_without_static                               { $$ = Node\NullableType[$2]; }
+    | union_type_without_static                             { $$ = Node\UnionType[$1]; }
+    | intersection_type_without_static                      { $$ = $1; }
+;
+
+optional_type_without_static:
       /* empty */                                           { $$ = null; }
-    | type_expr                                             { $$ = $1; }
+    | type_expr_without_static                              { $$ = $1; }
 ;
 
 optional_return_type:
       /* empty */                                           { $$ = null; }
     | ':' type_expr                                         { $$ = $2; }
+    | ':' error                                             { $$ = null; }
 ;
 
 argument_list:
       '(' ')'                                               { $$ = array(); }
-    | '(' non_empty_argument_list no_comma ')'              { $$ = $2; }
+    | '(' non_empty_argument_list optional_comma ')'        { $$ = $2; }
+    | '(' variadic_placeholder ')'                          { init($2); }
+;
+
+variadic_placeholder:
+      T_ELLIPSIS                                            { $$ = Node\VariadicPlaceholder[]; }
 ;
 
 non_empty_argument_list:
@@ -478,8 +674,10 @@ non_empty_argument_list:
 
 argument:
       expr                                                  { $$ = Node\Arg[$1, false, false]; }
-    | '&' variable                                          { $$ = Node\Arg[$2, true, false]; }
+    | ampersand variable                                    { $$ = Node\Arg[$2, true, false]; }
     | T_ELLIPSIS expr                                       { $$ = Node\Arg[$2, false, true]; }
+    | identifier_maybe_reserved ':' expr
+          { $$ = new Node\Arg($3, false, false, attributes(), $1); }
 ;
 
 global_var_list:
@@ -492,7 +690,7 @@ non_empty_global_var_list:
 ;
 
 global_var:
-      simple_variable                                       { $$ = Expr\Variable[$1]; }
+      simple_variable                                       { $$ = $1; }
 ;
 
 static_var_list:
@@ -509,20 +707,35 @@ static_var:
     | plain_variable '=' expr                               { $$ = Stmt\StaticVar[$1, $3]; }
 ;
 
-class_statement_list:
-      class_statement_list class_statement                  { push($1, $2); }
+class_statement_list_ex:
+      class_statement_list_ex class_statement               { if ($2 !== null) { push($1, $2); } else { $$ = $1; } }
     | /* empty */                                           { init(); }
 ;
 
+class_statement_list:
+      class_statement_list_ex
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
+            if ($nop !== null) { $1[] = $nop; } $$ = $1; }
+;
+
 class_statement:
-      variable_modifiers property_declaration_list ';'
-          { $$ = Stmt\Property[$1, $2]; $this->checkProperty($$, #1); }
-    | method_modifiers T_CONST class_const_list ';'
-          { $$ = Stmt\ClassConst[$3, $1]; $this->checkClassConst($$, #1); }
-    | method_modifiers T_FUNCTION optional_ref identifier_ex '(' parameter_list ')' optional_return_type method_body
-          { $$ = Stmt\ClassMethod[$4, ['type' => $1, 'byRef' => $3, 'params' => $6, 'returnType' => $8, 'stmts' => $9]];
-            $this->checkClassMethod($$, #1); }
+      optional_attributes variable_modifiers optional_type_without_static property_declaration_list semi
+          { $$ = new Stmt\Property($2, $4, attributes(), $3, $1);
+            $this->checkProperty($$, #2); }
+    | optional_attributes method_modifiers T_CONST class_const_list semi
+          { $$ = new Stmt\ClassConst($4, $2, attributes(), $1);
+            $this->checkClassConst($$, #2); }
+    | optional_attributes method_modifiers T_CONST type_expr class_const_list semi
+          { $$ = new Stmt\ClassConst($5, $2, attributes(), $1, $4);
+            $this->checkClassConst($$, #2); }
+    | optional_attributes method_modifiers T_FUNCTION optional_ref identifier_maybe_reserved '(' parameter_list ')'
+      optional_return_type method_body
+          { $$ = Stmt\ClassMethod[$5, ['type' => $2, 'byRef' => $4, 'params' => $7, 'returnType' => $9, 'stmts' => $10, 'attrGroups' => $1]];
+            $this->checkClassMethod($$, #2); }
     | T_USE class_name_list trait_adaptations               { $$ = Stmt\TraitUse[$2, $3]; }
+    | optional_attributes T_CASE identifier_maybe_reserved enum_case_expr semi
+         { $$ = Stmt\EnumCase[$3, $4, $1]; }
+    | error                                                 { $$ = null; /* will be skipped */ }
 ;
 
 trait_adaptations:
@@ -538,27 +751,27 @@ trait_adaptation_list:
 trait_adaptation:
       trait_method_reference_fully_qualified T_INSTEADOF class_name_list ';'
           { $$ = Stmt\TraitUseAdaptation\Precedence[$1[0], $1[1], $3]; }
-    | trait_method_reference T_AS member_modifier identifier_ex ';'
+    | trait_method_reference T_AS member_modifier identifier_maybe_reserved ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], $3, $4]; }
     | trait_method_reference T_AS member_modifier ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], $3, null]; }
-    | trait_method_reference T_AS identifier ';'
+    | trait_method_reference T_AS identifier_not_reserved ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], null, $3]; }
-    | trait_method_reference T_AS reserved_non_modifiers ';'
+    | trait_method_reference T_AS reserved_non_modifiers_identifier ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], null, $3]; }
 ;
 
 trait_method_reference_fully_qualified:
-      name T_PAAMAYIM_NEKUDOTAYIM identifier_ex             { $$ = array($1, $3); }
+      name T_PAAMAYIM_NEKUDOTAYIM identifier_maybe_reserved { $$ = array($1, $3); }
 ;
 trait_method_reference:
       trait_method_reference_fully_qualified                { $$ = $1; }
-    | identifier_ex                                         { $$ = array(null, $1); }
+    | identifier_maybe_reserved                             { $$ = array(null, $1); }
 ;
 
 method_body:
       ';' /* abstract method */                             { $$ = null; }
-    | '{' inner_statement_list '}'                          { $$ = $2; }
+    | block_or_error                                        { $$ = $1; }
 ;
 
 variable_modifiers:
@@ -583,6 +796,7 @@ member_modifier:
     | T_STATIC                                              { $$ = Stmt\Class_::MODIFIER_STATIC; }
     | T_ABSTRACT                                            { $$ = Stmt\Class_::MODIFIER_ABSTRACT; }
     | T_FINAL                                               { $$ = Stmt\Class_::MODIFIER_FINAL; }
+    | T_READONLY                                            { $$ = Stmt\Class_::MODIFIER_READONLY; }
 ;
 
 property_declaration_list:
@@ -604,10 +818,14 @@ property_declaration:
     | property_decl_name '=' expr                           { $$ = Stmt\PropertyProperty[$1, $3]; }
 ;
 
-expr_list:
+expr_list_forbid_comma:
       non_empty_expr_list no_comma                          { $$ = $1; }
 ;
 
+expr_list_allow_comma:
+      non_empty_expr_list optional_comma                    { $$ = $1; }
+;
+
 non_empty_expr_list:
       non_empty_expr_list ',' expr                          { push($1, $3); }
     | expr                                                  { init($1); }
@@ -615,7 +833,7 @@ non_empty_expr_list:
 
 for_expr:
       /* empty */                                           { $$ = array(); }
-    | expr_list                                             { $$ = $1; }
+    | expr_list_forbid_comma                                { $$ = $1; }
 ;
 
 expr:
@@ -623,8 +841,9 @@ expr:
     | list_expr '=' expr                                    { $$ = Expr\Assign[$1, $3]; }
     | array_short_syntax '=' expr                           { $$ = Expr\Assign[$1, $3]; }
     | variable '=' expr                                     { $$ = Expr\Assign[$1, $3]; }
-    | variable '=' '&' variable                             { $$ = Expr\AssignRef[$1, $4]; }
+    | variable '=' ampersand variable                       { $$ = Expr\AssignRef[$1, $4]; }
     | new_expr                                              { $$ = $1; }
+    | match                                                 { $$ = $1; }
     | T_CLONE expr                                          { $$ = Expr\Clone_[$2]; }
     | variable T_PLUS_EQUAL expr                            { $$ = Expr\AssignOp\Plus      [$1, $3]; }
     | variable T_MINUS_EQUAL expr                           { $$ = Expr\AssignOp\Minus     [$1, $3]; }
@@ -638,6 +857,7 @@ expr:
     | variable T_SL_EQUAL expr                              { $$ = Expr\AssignOp\ShiftLeft [$1, $3]; }
     | variable T_SR_EQUAL expr                              { $$ = Expr\AssignOp\ShiftRight[$1, $3]; }
     | variable T_POW_EQUAL expr                             { $$ = Expr\AssignOp\Pow       [$1, $3]; }
+    | variable T_COALESCE_EQUAL expr                        { $$ = Expr\AssignOp\Coalesce  [$1, $3]; }
     | variable T_INC                                        { $$ = Expr\PostInc[$1]; }
     | T_INC variable                                        { $$ = Expr\PreInc [$2]; }
     | variable T_DEC                                        { $$ = Expr\PostDec[$1]; }
@@ -648,7 +868,8 @@ expr:
     | expr T_LOGICAL_AND expr                               { $$ = Expr\BinaryOp\LogicalAnd[$1, $3]; }
     | expr T_LOGICAL_XOR expr                               { $$ = Expr\BinaryOp\LogicalXor[$1, $3]; }
     | expr '|' expr                                         { $$ = Expr\BinaryOp\BitwiseOr [$1, $3]; }
-    | expr '&' expr                                         { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | expr T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG expr   { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | expr T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG expr       { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
     | expr '^' expr                                         { $$ = Expr\BinaryOp\BitwiseXor[$1, $3]; }
     | expr '.' expr                                         { $$ = Expr\BinaryOp\Concat    [$1, $3]; }
     | expr '+' expr                                         { $$ = Expr\BinaryOp\Plus      [$1, $3]; }
@@ -677,7 +898,7 @@ expr:
     | expr '?' expr ':' expr                                { $$ = Expr\Ternary[$1, $3,   $5]; }
     | expr '?' ':' expr                                     { $$ = Expr\Ternary[$1, null, $4]; }
     | expr T_COALESCE expr                                  { $$ = Expr\BinaryOp\Coalesce[$1, $3]; }
-    | T_ISSET '(' variables_list ')'                        { $$ = Expr\Isset_[$3]; }
+    | T_ISSET '(' expr_list_allow_comma ')'                 { $$ = Expr\Isset_[$3]; }
     | T_EMPTY '(' expr ')'                                  { $$ = Expr\Empty_[$3]; }
     | T_INCLUDE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_INCLUDE]; }
     | T_INCLUDE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_INCLUDE_ONCE]; }
@@ -685,7 +906,10 @@ expr:
     | T_REQUIRE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE]; }
     | T_REQUIRE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE_ONCE]; }
     | T_INT_CAST expr                                       { $$ = Expr\Cast\Int_    [$2]; }
-    | T_DOUBLE_CAST expr                                    { $$ = Expr\Cast\Double  [$2]; }
+    | T_DOUBLE_CAST expr
+          { $attrs = attributes();
+            $attrs['kind'] = $this->getFloatCastKind($1);
+            $$ = new Expr\Cast\Double($2, $attrs); }
     | T_STRING_CAST expr                                    { $$ = Expr\Cast\String_ [$2]; }
     | T_ARRAY_CAST expr                                     { $$ = Expr\Cast\Array_  [$2]; }
     | T_OBJECT_CAST expr                                    { $$ = Expr\Cast\Object_ [$2]; }
@@ -703,17 +927,30 @@ expr:
     | T_YIELD expr                                          { $$ = Expr\Yield_[$2, null]; }
     | T_YIELD expr T_DOUBLE_ARROW expr                      { $$ = Expr\Yield_[$4, $2]; }
     | T_YIELD_FROM expr                                     { $$ = Expr\YieldFrom[$2]; }
-    | T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type
-      '{' inner_statement_list '}'
-          { $$ = Expr\Closure[['static' => false, 'byRef' => $2, 'params' => $4, 'uses' => $6, 'returnType' => $7, 'stmts' => $9]]; }
-    | T_STATIC T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type
-      '{' inner_statement_list '}'
-          { $$ = Expr\Closure[['static' => true, 'byRef' => $3, 'params' => $5, 'uses' => $7, 'returnType' => $8, 'stmts' => $10]]; }
+    | T_THROW expr                                          { $$ = Expr\Throw_[$2]; }
+
+    | T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr %prec T_THROW
+          { $$ = Expr\ArrowFunction[['static' => false, 'byRef' => $2, 'params' => $4, 'returnType' => $6, 'expr' => $8, 'attrGroups' => []]]; }
+    | T_STATIC T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr %prec T_THROW
+          { $$ = Expr\ArrowFunction[['static' => true, 'byRef' => $3, 'params' => $5, 'returnType' => $7, 'expr' => $9, 'attrGroups' => []]]; }
+    | T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type block_or_error
+          { $$ = Expr\Closure[['static' => false, 'byRef' => $2, 'params' => $4, 'uses' => $6, 'returnType' => $7, 'stmts' => $8, 'attrGroups' => []]]; }
+    | T_STATIC T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type       block_or_error
+          { $$ = Expr\Closure[['static' => true, 'byRef' => $3, 'params' => $5, 'uses' => $7, 'returnType' => $8, 'stmts' => $9, 'attrGroups' => []]]; }
+
+    | attributes T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr %prec T_THROW
+          { $$ = Expr\ArrowFunction[['static' => false, 'byRef' => $3, 'params' => $5, 'returnType' => $7, 'expr' => $9, 'attrGroups' => $1]]; }
+    | attributes T_STATIC T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr %prec T_THROW
+          { $$ = Expr\ArrowFunction[['static' => true, 'byRef' => $4, 'params' => $6, 'returnType' => $8, 'expr' => $10, 'attrGroups' => $1]]; }
+    | attributes T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type block_or_error
+          { $$ = Expr\Closure[['static' => false, 'byRef' => $3, 'params' => $5, 'uses' => $7, 'returnType' => $8, 'stmts' => $9, 'attrGroups' => $1]]; }
+    | attributes T_STATIC T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type       block_or_error
+          { $$ = Expr\Closure[['static' => true, 'byRef' => $4, 'params' => $6, 'uses' => $8, 'returnType' => $9, 'stmts' => $10, 'attrGroups' => $1]]; }
 ;
 
 anonymous_class:
-      T_CLASS ctor_arguments extends_from implements_list '{' class_statement_list '}'
-          { $$ = array(Stmt\Class_[null, ['type' => 0, 'extends' => $3, 'implements' => $4, 'stmts' => $6]], $2);
+      optional_attributes class_entry_type ctor_arguments extends_from implements_list '{' class_statement_list '}'
+          { $$ = array(Stmt\Class_[null, ['type' => $2, 'extends' => $4, 'implements' => $5, 'stmts' => $7, 'attrGroups' => $1]], $3);
             $this->checkClass($$[0], -1); }
 ;
 
@@ -729,7 +966,7 @@ lexical_vars:
 ;
 
 lexical_var_list:
-      non_empty_lexical_var_list no_comma                   { $$ = $1; }
+      non_empty_lexical_var_list optional_comma             { $$ = $1; }
 ;
 
 non_empty_lexical_var_list:
@@ -741,8 +978,13 @@ lexical_var:
       optional_ref plain_variable                           { $$ = Expr\ClosureUse[$2, $1]; }
 ;
 
+name_readonly:
+      T_READONLY                                            { $$ = Name[$1]; }
+;
+
 function_call:
       name argument_list                                    { $$ = Expr\FuncCall[$1, $2]; }
+    | name_readonly argument_list                           { $$ = Expr\FuncCall[$1, $2]; }
     | callable_expr argument_list                           { $$ = Expr\FuncCall[$1, $2]; }
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM member_name argument_list
           { $$ = Expr\StaticCall[$1, $3, $4]; }
@@ -754,20 +996,22 @@ class_name:
 ;
 
 name:
-      namespace_name_parts                                  { $$ = Name[$1]; }
-    | T_NS_SEPARATOR namespace_name_parts                   { $$ = Name\FullyQualified[$2]; }
-    | T_NAMESPACE T_NS_SEPARATOR namespace_name_parts       { $$ = Name\Relative[$3]; }
+      T_STRING                                              { $$ = Name[$1]; }
+    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
+    | T_NAME_FULLY_QUALIFIED                                { $$ = Name\FullyQualified[substr($1, 1)]; }
+    | T_NAME_RELATIVE                                       { $$ = Name\Relative[substr($1, 10)]; }
 ;
 
 class_name_reference:
       class_name                                            { $$ = $1; }
     | new_variable                                          { $$ = $1; }
+    | '(' expr ')'                                          { $$ = $2; }
     | error                                                 { $$ = Expr\Error[]; $this->errorState = 2; }
 ;
 
 class_name_or_var:
       class_name                                            { $$ = $1; }
-    | dereferencable                                        { $$ = $1; }
+    | fully_dereferencable                                  { $$ = $1; }
 ;
 
 exit_expr:
@@ -789,9 +1033,22 @@ ctor_arguments:
 
 constant:
       name                                                  { $$ = Expr\ConstFetch[$1]; }
-    | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM identifier_ex
+    | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }
+    | T_FILE                                                { $$ = Scalar\MagicConst\File[]; }
+    | T_DIR                                                 { $$ = Scalar\MagicConst\Dir[]; }
+    | T_CLASS_C                                             { $$ = Scalar\MagicConst\Class_[]; }
+    | T_TRAIT_C                                             { $$ = Scalar\MagicConst\Trait_[]; }
+    | T_METHOD_C                                            { $$ = Scalar\MagicConst\Method[]; }
+    | T_FUNC_C                                              { $$ = Scalar\MagicConst\Function_[]; }
+    | T_NS_C                                                { $$ = Scalar\MagicConst\Namespace_[]; }
+;
+
+class_constant:
+      class_name_or_var T_PAAMAYIM_NEKUDOTAYIM identifier_maybe_reserved
           { $$ = Expr\ClassConstFetch[$1, $3]; }
-    /* We interpret and isolated FOO:: as an unfinished class constant fetch. It could also be
+    | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM '{' expr '}'
+          { $$ = Expr\ClassConstFetch[$1, $4]; }
+    /* We interpret an isolated FOO:: as an unfinished class constant fetch. It could also be
        an unfinished static property fetch or unfinished scoped call. */
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM error
           { $$ = Expr\ClassConstFetch[$1, new Expr\Error(stackAttributes(#3))]; $this->errorState = 2; }
@@ -808,36 +1065,24 @@ dereferencable_scalar:
           { $attrs = attributes(); $attrs['kind'] = Expr\Array_::KIND_LONG;
             $$ = new Expr\Array_($3, $attrs); }
     | array_short_syntax                                    { $$ = $1; }
-    | T_CONSTANT_ENCAPSED_STRING
-          { $attrs = attributes(); $attrs['kind'] = strKind($1);
-            $$ = new Scalar\String_(Scalar\String_::parse($1), $attrs); }
+    | T_CONSTANT_ENCAPSED_STRING                            { $$ = Scalar\String_::fromString($1, attributes()); }
+    | '"' encaps_list '"'
+          { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
+            parseEncapsed($2, '"', true); $$ = new Scalar\Encapsed($2, $attrs); }
 ;
 
 scalar:
       T_LNUMBER                                             { $$ = $this->parseLNumber($1, attributes()); }
-    | T_DNUMBER                                             { $$ = Scalar\DNumber[Scalar\DNumber::parse($1)]; }
-    | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }
-    | T_FILE                                                { $$ = Scalar\MagicConst\File[]; }
-    | T_DIR                                                 { $$ = Scalar\MagicConst\Dir[]; }
-    | T_CLASS_C                                             { $$ = Scalar\MagicConst\Class_[]; }
-    | T_TRAIT_C                                             { $$ = Scalar\MagicConst\Trait_[]; }
-    | T_METHOD_C                                            { $$ = Scalar\MagicConst\Method[]; }
-    | T_FUNC_C                                              { $$ = Scalar\MagicConst\Function_[]; }
-    | T_NS_C                                                { $$ = Scalar\MagicConst\Namespace_[]; }
+    | T_DNUMBER                                             { $$ = Scalar\DNumber::fromString($1, attributes()); }
     | dereferencable_scalar                                 { $$ = $1; }
     | constant                                              { $$ = $1; }
+    | class_constant                                        { $$ = $1; }
     | T_START_HEREDOC T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC
-          { $attrs = attributes(); setDocStringAttrs($attrs, $1);
-            $$ = new Scalar\String_(Scalar\String_::parseDocString($1, $2), $attrs); }
+          { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), true); }
     | T_START_HEREDOC T_END_HEREDOC
-          { $attrs = attributes(); setDocStringAttrs($attrs, $1);
-            $$ = new Scalar\String_('', $attrs); }
-    | '"' encaps_list '"'
-          { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
-            parseEncapsed($2, '"', true); $$ = new Scalar\Encapsed($2, $attrs); }
+          { $$ = $this->parseDocString($1, '', $2, attributes(), stackAttributes(#2), true); }
     | T_START_HEREDOC encaps_list T_END_HEREDOC
-          { $attrs = attributes(); setDocStringAttrs($attrs, $1);
-            parseEncapsedDoc($2, true); $$ = new Scalar\Encapsed($2, $attrs); }
+          { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), true); }
 ;
 
 optional_expr:
@@ -845,10 +1090,16 @@ optional_expr:
     | expr                                                  { $$ = $1; }
 ;
 
-dereferencable:
+fully_dereferencable:
       variable                                              { $$ = $1; }
     | '(' expr ')'                                          { $$ = $2; }
     | dereferencable_scalar                                 { $$ = $1; }
+    | class_constant                                        { $$ = $1; }
+;
+
+array_object_dereferencable:
+      fully_dereferencable                                  { $$ = $1; }
+    | constant                                              { $$ = $1; }
 ;
 
 callable_expr:
@@ -858,31 +1109,40 @@ callable_expr:
 ;
 
 callable_variable:
-      simple_variable                                       { $$ = Expr\Variable[$1]; }
-    | dereferencable '[' optional_expr ']'                  { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | constant '[' optional_expr ']'                        { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | dereferencable '{' expr '}'                           { $$ = Expr\ArrayDimFetch[$1, $3]; }
+      simple_variable                                       { $$ = $1; }
+    | array_object_dereferencable '[' optional_expr ']'     { $$ = Expr\ArrayDimFetch[$1, $3]; }
+    | array_object_dereferencable '{' expr '}'              { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | function_call                                         { $$ = $1; }
-    | dereferencable T_OBJECT_OPERATOR property_name argument_list
+    | array_object_dereferencable T_OBJECT_OPERATOR property_name argument_list
           { $$ = Expr\MethodCall[$1, $3, $4]; }
+    | array_object_dereferencable T_NULLSAFE_OBJECT_OPERATOR property_name argument_list
+          { $$ = Expr\NullsafeMethodCall[$1, $3, $4]; }
+;
+
+optional_plain_variable:
+      /* empty */                                           { $$ = null; }
+    | plain_variable                                        { $$ = $1; }
 ;
 
 variable:
       callable_variable                                     { $$ = $1; }
     | static_member                                         { $$ = $1; }
-    | dereferencable T_OBJECT_OPERATOR property_name        { $$ = Expr\PropertyFetch[$1, $3]; }
+    | array_object_dereferencable T_OBJECT_OPERATOR property_name
+          { $$ = Expr\PropertyFetch[$1, $3]; }
+    | array_object_dereferencable T_NULLSAFE_OBJECT_OPERATOR property_name
+          { $$ = Expr\NullsafePropertyFetch[$1, $3]; }
 ;
 
 simple_variable:
-      T_VARIABLE                                            { $$ = parseVar($1); }
-    | '$' '{' expr '}'                                      { $$ = $3; }
+      plain_variable                                        { $$ = $1; }
+    | '$' '{' expr '}'                                      { $$ = Expr\Variable[$3]; }
     | '$' simple_variable                                   { $$ = Expr\Variable[$2]; }
-    | '$' error                                             { $$ = Expr\Error[]; $this->errorState = 2; }
+    | '$' error                                             { $$ = Expr\Variable[Expr\Error[]]; $this->errorState = 2; }
 ;
 
 static_member_prop_name:
       simple_variable
-          { $var = $1; $$ = \is_string($var) ? Node\VarLikeIdentifier[$var] : $var; }
+          { $var = $1->name; $$ = \is_string($var) ? Node\VarLikeIdentifier[$var] : $var; }
 ;
 
 static_member:
@@ -891,10 +1151,11 @@ static_member:
 ;
 
 new_variable:
-      simple_variable                                       { $$ = Expr\Variable[$1]; }
+      simple_variable                                       { $$ = $1; }
     | new_variable '[' optional_expr ']'                    { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | new_variable '{' expr '}'                             { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | new_variable T_OBJECT_OPERATOR property_name          { $$ = Expr\PropertyFetch[$1, $3]; }
+    | new_variable T_NULLSAFE_OBJECT_OPERATOR property_name { $$ = Expr\NullsafePropertyFetch[$1, $3]; }
     | class_name T_PAAMAYIM_NEKUDOTAYIM static_member_prop_name
           { $$ = Expr\StaticPropertyFetch[$1, $3]; }
     | new_variable T_PAAMAYIM_NEKUDOTAYIM static_member_prop_name
@@ -902,50 +1163,46 @@ new_variable:
 ;
 
 member_name:
-      identifier_ex                                         { $$ = $1; }
-    | '{' expr '}'	                                        { $$ = $2; }
-    | simple_variable	                                    { $$ = Expr\Variable[$1]; }
+      identifier_maybe_reserved                             { $$ = $1; }
+    | '{' expr '}'                                          { $$ = $2; }
+    | simple_variable                                       { $$ = $1; }
 ;
 
 property_name:
-      identifier                                            { $$ = $1; }
-    | '{' expr '}'	                                        { $$ = $2; }
-    | simple_variable	                                    { $$ = Expr\Variable[$1]; }
+      identifier_not_reserved                               { $$ = $1; }
+    | '{' expr '}'                                          { $$ = $2; }
+    | simple_variable                                       { $$ = $1; }
     | error                                                 { $$ = Expr\Error[]; $this->errorState = 2; }
 ;
 
 list_expr:
-      T_LIST '(' list_expr_elements ')'                     { $$ = Expr\List_[$3]; }
-;
-
-list_expr_elements:
-      list_expr_elements ',' list_expr_element              { push($1, $3); }
-    | list_expr_element                                     { init($1); }
-;
-
-list_expr_element:
-      variable                                              { $$ = Expr\ArrayItem[$1, null, false]; }
-    | list_expr                                             { $$ = Expr\ArrayItem[$1, null, false]; }
-    | expr T_DOUBLE_ARROW variable                          { $$ = Expr\ArrayItem[$3, $1, false]; }
-    | expr T_DOUBLE_ARROW list_expr                         { $$ = Expr\ArrayItem[$3, $1, false]; }
-    | /* empty */                                           { $$ = null; }
+      T_LIST '(' inner_array_pair_list ')'                  { $$ = Expr\List_[$3]; }
 ;
 
 array_pair_list:
       inner_array_pair_list
-          { $$ = $1; $end = count($$)-1; if ($$[$end] === null) unset($$[$end]); }
+          { $$ = $1; $end = count($$)-1; if ($$[$end] === null) array_pop($$); }
+;
+
+comma_or_error:
+      ','
+    | error
+          { /* do nothing -- prevent default action of $$=$1. See #551. */ }
 ;
 
 inner_array_pair_list:
-      inner_array_pair_list ',' array_pair                  { push($1, $3); }
+      inner_array_pair_list comma_or_error array_pair       { push($1, $3); }
     | array_pair                                            { init($1); }
 ;
 
 array_pair:
-      expr T_DOUBLE_ARROW expr                              { $$ = Expr\ArrayItem[$3, $1,   false]; }
-    | expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
-    | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
-    | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
+      expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
+    | ampersand variable                                    { $$ = Expr\ArrayItem[$2, null, true]; }
+    | list_expr                                             { $$ = Expr\ArrayItem[$1, null, false]; }
+    | expr T_DOUBLE_ARROW expr                              { $$ = Expr\ArrayItem[$3, $1,   false]; }
+    | expr T_DOUBLE_ARROW ampersand variable                { $$ = Expr\ArrayItem[$4, $1,   true]; }
+    | expr T_DOUBLE_ARROW list_expr                         { $$ = Expr\ArrayItem[$3, $1,   false]; }
+    | T_ELLIPSIS expr                                       { $$ = new Expr\ArrayItem($2, null, false, attributes(), true); }
     | /* empty */                                           { $$ = null; }
 ;
 
@@ -967,7 +1224,10 @@ encaps_str_varname:
 encaps_var:
       plain_variable                                        { $$ = $1; }
     | plain_variable '[' encaps_var_offset ']'              { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | plain_variable T_OBJECT_OPERATOR identifier           { $$ = Expr\PropertyFetch[$1, $3]; }
+    | plain_variable T_OBJECT_OPERATOR identifier_not_reserved
+          { $$ = Expr\PropertyFetch[$1, $3]; }
+    | plain_variable T_NULLSAFE_OBJECT_OPERATOR identifier_not_reserved
+          { $$ = Expr\NullsafePropertyFetch[$1, $3]; }
     | T_DOLLAR_OPEN_CURLY_BRACES expr '}'                   { $$ = Expr\Variable[$2]; }
     | T_DOLLAR_OPEN_CURLY_BRACES T_STRING_VARNAME '}'       { $$ = Expr\Variable[$2]; }
     | T_DOLLAR_OPEN_CURLY_BRACES encaps_str_varname '[' expr ']' '}'

grammar/phpyLang.php

@@ -0,0 +1,184 @@
+<?php
+
+///////////////////////////////
+/// Utility regex constants ///
+///////////////////////////////
+
+const LIB = '(?(DEFINE)
+    (?<singleQuotedString>\'[^\\\\\']*+(?:\\\\.[^\\\\\']*+)*+\')
+    (?<doubleQuotedString>"[^\\\\"]*+(?:\\\\.[^\\\\"]*+)*+")
+    (?<string>(?&singleQuotedString)|(?&doubleQuotedString))
+    (?<comment>/\*[^*]*+(?:\*(?!/)[^*]*+)*+\*/)
+    (?<code>\{[^\'"/{}]*+(?:(?:(?&string)|(?&comment)|(?&code)|/)[^\'"/{}]*+)*+})
+)';
+
+const PARAMS = '\[(?<params>[^[\]]*+(?:\[(?&params)\][^[\]]*+)*+)\]';
+const ARGS   = '\((?<args>[^()]*+(?:\((?&args)\)[^()]*+)*+)\)';
+
+///////////////////////////////
+/// Preprocessing functions ///
+///////////////////////////////
+
+function preprocessGrammar($code) {
+    $code = resolveNodes($code);
+    $code = resolveMacros($code);
+    $code = resolveStackAccess($code);
+
+    return $code;
+}
+
+function resolveNodes($code) {
+    return preg_replace_callback(
+        '~\b(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
+        function($matches) {
+            // recurse
+            $matches['params'] = resolveNodes($matches['params']);
+
+            $params = magicSplit(
+                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
+                $matches['params']
+            );
+
+            $paramCode = '';
+            foreach ($params as $param) {
+                $paramCode .= $param . ', ';
+            }
+
+            return 'new ' . $matches['name'] . '(' . $paramCode . 'attributes())';
+        },
+        $code
+    );
+}
+
+function resolveMacros($code) {
+    return preg_replace_callback(
+        '~\b(?<!::|->)(?!array\()(?<name>[a-z][A-Za-z]++)' . ARGS . '~',
+        function($matches) {
+            // recurse
+            $matches['args'] = resolveMacros($matches['args']);
+
+            $name = $matches['name'];
+            $args = magicSplit(
+                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
+                $matches['args']
+            );
+
+            if ('attributes' === $name) {
+                assertArgs(0, $args, $name);
+                return '$this->startAttributeStack[#1] + $this->endAttributes';
+            }
+
+            if ('stackAttributes' === $name) {
+                assertArgs(1, $args, $name);
+                return '$this->startAttributeStack[' . $args[0] . ']'
+                       . ' + $this->endAttributeStack[' . $args[0] . ']';
+            }
+
+            if ('init' === $name) {
+                return '$$ = array(' . implode(', ', $args) . ')';
+            }
+
+            if ('push' === $name) {
+                assertArgs(2, $args, $name);
+
+                return $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0];
+            }
+
+            if ('pushNormalizing' === $name) {
+                assertArgs(2, $args, $name);
+
+                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
+                       . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
+            }
+
+            if ('toArray' == $name) {
+                assertArgs(1, $args, $name);
+
+                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
+            }
+
+            if ('parseVar' === $name) {
+                assertArgs(1, $args, $name);
+
+                return 'substr(' . $args[0] . ', 1)';
+            }
+
+            if ('parseEncapsed' === $name) {
+                assertArgs(3, $args, $name);
+
+                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\Scalar\EncapsedStringPart) {'
+                       . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, ' . $args[1] . ', ' . $args[2] . '); } }';
+            }
+
+            if ('makeNop' === $name) {
+                assertArgs(3, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                       . ' if (isset($startAttributes[\'comments\']))'
+                       . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
+                       . ' else { ' . $args[0] . ' = null; }';
+            }
+
+            if ('makeZeroLengthNop' == $name) {
+                assertArgs(2, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                       . ' if (isset($startAttributes[\'comments\']))'
+                       . ' { ' . $args[0] . ' = new Stmt\Nop($this->createCommentNopAttributes($startAttributes[\'comments\'])); }'
+                       . ' else { ' . $args[0] . ' = null; }';
+            }
+
+            if ('prependLeadingComments' === $name) {
+                assertArgs(1, $args, $name);
+
+                return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
+                       . 'if (!empty($attrs[\'comments\'])) {'
+                       . '$stmts[0]->setAttribute(\'comments\', '
+                       . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
+            }
+
+            return $matches[0];
+        },
+        $code
+    );
+}
+
+function assertArgs($num, $args, $name) {
+    if ($num != count($args)) {
+        die('Wrong argument count for ' . $name . '().');
+    }
+}
+
+function resolveStackAccess($code) {
+    $code = preg_replace('/\$\d+/', '$this->semStack[$0]', $code);
+    $code = preg_replace('/#(\d+)/', '$$1', $code);
+    return $code;
+}
+
+function removeTrailingWhitespace($code) {
+    $lines = explode("\n", $code);
+    $lines = array_map('rtrim', $lines);
+    return implode("\n", $lines);
+}
+
+//////////////////////////////
+/// Regex helper functions ///
+//////////////////////////////
+
+function regex($regex) {
+    return '~' . LIB . '(?:' . str_replace('~', '\~', $regex) . ')~';
+}
+
+function magicSplit($regex, $string) {
+    $pieces = preg_split(regex('(?:(?&string)|(?&comment)|(?&code))(*SKIP)(*FAIL)|' . $regex), $string);
+
+    foreach ($pieces as &$piece) {
+        $piece = trim($piece);
+    }
+
+    if ($pieces === ['']) {
+        return [];
+    }
+
+    return $pieces;
+}

grammar/rebuildParsers.php

@@ -1,5 +1,7 @@
 <?php
 
+require __DIR__ . '/phpyLang.php';
+
 $grammarFileToName = [
     __DIR__ . '/php5.y' => 'Php5',
     __DIR__ . '/php7.y' => 'Php7',
@@ -13,31 +15,16 @@ $tmpResultFile  = __DIR__ . '/tmp_parser.php';
 $resultDir = __DIR__ . '/../lib/PhpParser/Parser';
 $tokensResultsFile = $resultDir . '/Tokens.php';
 
-// check for kmyacc.exe binary in this directory, otherwise fall back to global name
-$kmyacc = __DIR__ . '/kmyacc.exe';
-if (!file_exists($kmyacc)) {
-    $kmyacc = 'kmyacc';
+$kmyacc = getenv('KMYACC');
+if (!$kmyacc) {
+    // Use phpyacc from dev dependencies by default.
+    $kmyacc = __DIR__ . '/../vendor/bin/phpyacc';
 }
 
 $options = array_flip($argv);
 $optionDebug = isset($options['--debug']);
 $optionKeepTmpGrammar = isset($options['--keep-tmp-grammar']);
 
-///////////////////////////////
-/// Utility regex constants ///
-///////////////////////////////
-
-const LIB = '(?(DEFINE)
-    (?<singleQuotedString>\'[^\\\\\']*+(?:\\\\.[^\\\\\']*+)*+\')
-    (?<doubleQuotedString>"[^\\\\"]*+(?:\\\\.[^\\\\"]*+)*+")
-    (?<string>(?&singleQuotedString)|(?&doubleQuotedString))
-    (?<comment>/\*[^*]*+(?:\*(?!/)[^*]*+)*+\*/)
-    (?<code>\{[^\'"/{}]*+(?:(?:(?&string)|(?&comment)|(?&code)|/)[^\'"/{}]*+)*+})
-)';
-
-const PARAMS = '\[(?<params>[^[\]]*+(?:\[(?&params)\][^[\]]*+)*+)\]';
-const ARGS   = '\((?<args>[^()]*+(?:\((?&args)\)[^()]*+)*+)\)';
-
 ///////////////////
 /// Main script ///
 ///////////////////
@@ -49,18 +36,14 @@ foreach ($grammarFileToName as $grammarFile => $name) {
 
     $grammarCode = file_get_contents($grammarFile);
     $grammarCode = str_replace('%tokens', $tokens, $grammarCode);
-
-    $grammarCode = resolveNodes($grammarCode);
-    $grammarCode = resolveMacros($grammarCode);
-    $grammarCode = resolveStackAccess($grammarCode);
+    $grammarCode = preprocessGrammar($grammarCode);
 
     file_put_contents($tmpGrammarFile, $grammarCode);
 
     $additionalArgs = $optionDebug ? '-t -v' : '';
 
     echo "Building $name parser.\n";
-    $output = trim(shell_exec("$kmyacc $additionalArgs -l -m $skeletonFile -p $name $tmpGrammarFile 2>&1"));
-    echo "Output: \"$output\"\n";
+    $output = execCmd("$kmyacc $additionalArgs -m $skeletonFile -p $name $tmpGrammarFile");
 
     $resultCode = file_get_contents($tmpResultFile);
     $resultCode = removeTrailingWhitespace($resultCode);
@@ -70,8 +53,7 @@ foreach ($grammarFileToName as $grammarFile => $name) {
     unlink($tmpResultFile);
 
     echo "Building token definition.\n";
-    $output = trim(shell_exec("$kmyacc -l -m $tokensTemplate $tmpGrammarFile 2>&1"));
-    assert($output === '');
+    $output = execCmd("$kmyacc -m $tokensTemplate $tmpGrammarFile");
     rename($tmpResultFile, $tokensResultsFile);
 
     if (!$optionKeepTmpGrammar) {
@@ -79,160 +61,9 @@ foreach ($grammarFileToName as $grammarFile => $name) {
     }
 }
 
-///////////////////////////////
-/// Preprocessing functions ///
-///////////////////////////////
-
-function resolveNodes($code) {
-    return preg_replace_callback(
-        '~\b(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
-        function($matches) {
-            // recurse
-            $matches['params'] = resolveNodes($matches['params']);
-
-            $params = magicSplit(
-                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
-                $matches['params']
-            );
-
-            $paramCode = '';
-            foreach ($params as $param) {
-                $paramCode .= $param . ', ';
-            }
-
-            return 'new ' . $matches['name'] . '(' . $paramCode . 'attributes())';
-        },
-        $code
-    );
-}
-
-function resolveMacros($code) {
-    return preg_replace_callback(
-        '~\b(?<!::|->)(?!array\()(?<name>[a-z][A-Za-z]++)' . ARGS . '~',
-        function($matches) {
-            // recurse
-            $matches['args'] = resolveMacros($matches['args']);
-
-            $name = $matches['name'];
-            $args = magicSplit(
-                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
-                $matches['args']
-            );
-
-            if ('attributes' == $name) {
-                assertArgs(0, $args, $name);
-                return '$this->startAttributeStack[#1] + $this->endAttributes';
-            }
-
-            if ('stackAttributes' == $name) {
-                assertArgs(1, $args, $name);
-                return '$this->startAttributeStack[' . $args[0] . ']'
-                     . ' + $this->endAttributeStack[' . $args[0] . ']';
-            }
-
-            if ('init' == $name) {
-                return '$$ = array(' . implode(', ', $args) . ')';
-            }
-
-            if ('push' == $name) {
-                assertArgs(2, $args, $name);
-
-                return $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0];
-            }
-
-            if ('pushNormalizing' == $name) {
-                assertArgs(2, $args, $name);
-
-                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
-                     . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
-            }
-
-            if ('toArray' == $name) {
-                assertArgs(1, $args, $name);
-
-                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
-            }
-
-            if ('parseVar' == $name) {
-                assertArgs(1, $args, $name);
-
-                return 'substr(' . $args[0] . ', 1)';
-            }
-
-            if ('parseEncapsed' == $name) {
-                assertArgs(3, $args, $name);
-
-                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\Scalar\EncapsedStringPart) {'
-                     . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, ' . $args[1] . ', ' . $args[2] . '); } }';
-            }
-
-            if ('parseEncapsedDoc' == $name) {
-                assertArgs(2, $args, $name);
-
-                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\Scalar\EncapsedStringPart) {'
-                     . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, null, ' . $args[1] . '); } }'
-                     . ' $s->value = preg_replace(\'~(\r\n|\n|\r)\z~\', \'\', $s->value);'
-                     . ' if (\'\' === $s->value) array_pop(' . $args[0] . ');';
-            }
-
-            if ('makeNop' == $name) {
-                assertArgs(3, $args, $name);
-
-                return '$startAttributes = ' . $args[1] . ';'
-                . ' if (isset($startAttributes[\'comments\']))'
-                . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
-                . ' else { ' . $args[0] . ' = null; }';
-            }
-
-            if ('strKind' == $name) {
-                assertArgs(1, $args, $name);
-
-                return '(' . $args[0] . '[0] === "\'" || (' . $args[0] . '[1] === "\'" && '
-                     . '(' . $args[0] . '[0] === \'b\' || ' . $args[0] . '[0] === \'B\')) '
-                     . '? Scalar\String_::KIND_SINGLE_QUOTED : Scalar\String_::KIND_DOUBLE_QUOTED)';
-            }
-
-            if ('setDocStringAttrs' == $name) {
-                assertArgs(2, $args, $name);
-
-                return $args[0] . '[\'kind\'] = strpos(' . $args[1] . ', "\'") === false '
-                     . '? Scalar\String_::KIND_HEREDOC : Scalar\String_::KIND_NOWDOC; '
-                     . 'preg_match(\'/\A[bB]?<<<[ \t]*[\\\'"]?([a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*)[\\\'"]?(?:\r\n|\n|\r)\z/\', ' . $args[1] . ', $matches); '
-                     . $args[0] . '[\'docLabel\'] = $matches[1];';
-            }
-
-            if ('prependLeadingComments' == $name) {
-                assertArgs(1, $args, $name);
-
-                return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
-                . 'if (!empty($attrs[\'comments\'])) {'
-                . '$stmts[0]->setAttribute(\'comments\', '
-                . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
-            }
-
-            return $matches[0];
-        },
-        $code
-    );
-}
-
-function assertArgs($num, $args, $name) {
-    if ($num != count($args)) {
-        die('Wrong argument count for ' . $name . '().');
-    }
-}
-
-function resolveStackAccess($code) {
-    $code = preg_replace('/\$\d+/', '$this->semStack[$0]', $code);
-    $code = preg_replace('/#(\d+)/', '$$1', $code);
-    return $code;
-}
-
-function removeTrailingWhitespace($code) {
-    $lines = explode("\n", $code);
-    $lines = array_map('rtrim', $lines);
-    return implode("\n", $lines);
-}
+////////////////////////////////
+/// Utility helper functions ///
+////////////////////////////////
 
 function ensureDirExists($dir) {
     if (!is_dir($dir)) {
@@ -240,24 +71,11 @@ function ensureDirExists($dir) {
     }
 }
 
-//////////////////////////////
-/// Regex helper functions ///
-//////////////////////////////
-
-function regex($regex) {
-    return '~' . LIB . '(?:' . str_replace('~', '\~', $regex) . ')~';
-}
-
-function magicSplit($regex, $string) {
-    $pieces = preg_split(regex('(?:(?&string)|(?&comment)|(?&code))(*SKIP)(*FAIL)|' . $regex), $string);
-
-    foreach ($pieces as &$piece) {
-        $piece = trim($piece);
+function execCmd($cmd) {
+    $output = trim(shell_exec("$cmd 2>&1"));
+    if ($output !== "") {
+        echo "> " . $cmd . "\n";
+        echo $output;
     }
-
-    if ($pieces === ['']) {
-        return [];
-    }
-
-    return $pieces;
+    return $output;
 }

grammar/tokens.y

@@ -1,6 +1,7 @@
 /* We currently rely on the token ID mapping to be the same between PHP 5 and PHP 7 - so the same lexer can be used for
  * both. This is enforced by sharing this token file. */
 
+%right T_THROW
 %left T_INCLUDE T_INCLUDE_ONCE T_EVAL T_REQUIRE T_REQUIRE_ONCE
 %left ','
 %left T_LOGICAL_OR
@@ -10,14 +11,14 @@
 %right T_YIELD
 %right T_DOUBLE_ARROW
 %right T_YIELD_FROM
-%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL
+%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL T_COALESCE_EQUAL
 %left '?' ':'
 %right T_COALESCE
 %left T_BOOLEAN_OR
 %left T_BOOLEAN_AND
 %left '|'
 %left '^'
-%left '&'
+%left T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
 %nonassoc T_IS_EQUAL T_IS_NOT_EQUAL T_IS_IDENTICAL T_IS_NOT_IDENTICAL T_SPACESHIP
 %nonassoc '<' T_IS_SMALLER_OR_EQUAL '>' T_IS_GREATER_OR_EQUAL
 %left T_SL T_SR
@@ -41,8 +42,6 @@
 %token T_VARIABLE
 %token T_NUM_STRING
 %token T_INLINE_HTML
-%token T_CHARACTER
-%token T_BAD_CHARACTER
 %token T_ENCAPSED_AND_WHITESPACE
 %token T_CONSTANT_ENCAPSED_STRING
 %token T_ECHO
@@ -57,6 +56,7 @@
 %token T_ENDDECLARE
 %token T_AS
 %token T_SWITCH
+%token T_MATCH
 %token T_ENDSWITCH
 %token T_CASE
 %token T_DEFAULT
@@ -64,6 +64,7 @@
 %token T_CONTINUE
 %token T_GOTO
 %token T_FUNCTION
+%token T_FN
 %token T_CONST
 %token T_RETURN
 %token T_TRY
@@ -73,7 +74,7 @@
 %token T_USE
 %token T_INSTEADOF
 %token T_GLOBAL
-%right T_STATIC T_ABSTRACT T_FINAL T_PRIVATE T_PROTECTED T_PUBLIC
+%right T_STATIC T_ABSTRACT T_FINAL T_PRIVATE T_PROTECTED T_PUBLIC T_READONLY
 %token T_VAR
 %token T_UNSET
 %token T_ISSET
@@ -82,9 +83,11 @@
 %token T_CLASS
 %token T_TRAIT
 %token T_INTERFACE
+%token T_ENUM
 %token T_EXTENDS
 %token T_IMPLEMENTS
 %token T_OBJECT_OPERATOR
+%token T_NULLSAFE_OBJECT_OPERATOR
 %token T_DOUBLE_ARROW
 %token T_LIST
 %token T_ARRAY
@@ -95,12 +98,6 @@
 %token T_FUNC_C
 %token T_LINE
 %token T_FILE
-%token T_COMMENT
-%token T_DOC_COMMENT
-%token T_OPEN_TAG
-%token T_OPEN_TAG_WITH_ECHO
-%token T_CLOSE_TAG
-%token T_WHITESPACE
 %token T_START_HEREDOC
 %token T_END_HEREDOC
 %token T_DOLLAR_OPEN_CURLY_BRACES
@@ -111,3 +108,8 @@
 %token T_DIR
 %token T_NS_SEPARATOR
 %token T_ELLIPSIS
+%token T_NAME_FULLY_QUALIFIED
+%token T_NAME_QUALIFIED
+%token T_NAME_RELATIVE
+%token T_ATTRIBUTE
+%token T_ENUM

lib/PhpParser/Autoloader.php

@@ -1,40 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser;
-
-/**
- * @codeCoverageIgnore
- */
-class Autoloader
-{
-    /** @var bool Whether the autoloader has been registered. */
-    private static $registered = false;
-
-    /**
-     * Registers PhpParser\Autoloader as an SPL autoloader.
-     *
-     * @param bool $prepend Whether to prepend the autoloader instead of appending
-     */
-    static public function register(bool $prepend = false) {
-        if (self::$registered === true) {
-            return;
-        }
-
-        spl_autoload_register([__CLASS__, 'autoload'], true, $prepend);
-        self::$registered = true;
-    }
-
-    /**
-     * Handles autoloading of classes.
-     *
-     * @param string $class A class name.
-     */
-    static public function autoload(string $class) {
-        if (0 === strpos($class, 'PhpParser\\')) {
-            $fileName = __DIR__ . strtr(substr($class, 9), '\\', '/') . '.php';
-            if (file_exists($fileName)) {
-                require $fileName;
-            }
-        }
-    }
-}

lib/PhpParser/Builder.php

@@ -10,4 +10,4 @@ interface Builder
      * @return Node The built node
      */
     public function getNode() : Node;
-}
+}

lib/PhpParser/Builder/ClassConst.php

@@ -0,0 +1,148 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PhpParser\Builder;
+
+use PhpParser;
+use PhpParser\BuilderHelpers;
+use PhpParser\Node;
+use PhpParser\Node\Const_;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Stmt;
+
+class ClassConst implements PhpParser\Builder
+{
+    protected $flags = 0;
+    protected $attributes = [];
+    protected $constants = [];
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+    /** @var Identifier|Node\Name|Node\ComplexType */
+    protected $type;
+
+    /**
+     * Creates a class constant builder
+     *
+     * @param string|Identifier                          $name  Name
+     * @param Node\Expr|bool|null|int|float|string|array $value Value
+     */
+    public function __construct($name, $value) {
+        $this->constants = [new Const_($name, BuilderHelpers::normalizeValue($value))];
+    }
+
+    /**
+     * Add another constant to const group
+     *
+     * @param string|Identifier                          $name  Name
+     * @param Node\Expr|bool|null|int|float|string|array $value Value
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addConst($name, $value) {
+        $this->constants[] = new Const_($name, BuilderHelpers::normalizeValue($value));
+
+        return $this;
+    }
+
+    /**
+     * Makes the constant public.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makePublic() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PUBLIC);
+
+        return $this;
+    }
+
+    /**
+     * Makes the constant protected.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeProtected() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PROTECTED);
+
+        return $this;
+    }
+
+    /**
+     * Makes the constant private.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makePrivate() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PRIVATE);
+
+        return $this;
+    }
+
+    /**
+     * Makes the constant final.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeFinal() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
+
+        return $this;
+    }
+
+    /**
+     * Sets doc comment for the constant.
+     *
+     * @param PhpParser\Comment\Doc|string $docComment Doc comment to set
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function setDocComment($docComment) {
+        $this->attributes = [
+            'comments' => [BuilderHelpers::normalizeDocComment($docComment)]
+        ];
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
+    /**
+     * Sets the constant type.
+     *
+     * @param string|Node\Name|Identifier|Node\ComplexType $type
+     *
+     * @return $this
+     */
+    public function setType($type) {
+        $this->type = BuilderHelpers::normalizeType($type);
+
+        return $this;
+    }
+
+    /**
+     * Returns the built class node.
+     *
+     * @return Stmt\ClassConst The built constant node
+     */
+    public function getNode(): PhpParser\Node {
+        return new Stmt\ClassConst(
+            $this->constants,
+            $this->flags,
+            $this->attributes,
+            $this->attributeGroups,
+            $this->type
+        );
+    }
+}

lib/PhpParser/Builder/Class_.php

@@ -4,6 +4,7 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Node;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
@@ -20,6 +21,9 @@ class Class_ extends Declaration
     protected $properties = [];
     protected $methods = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a class builder.
      *
@@ -63,7 +67,7 @@ class Class_ extends Declaration
      * @return $this The builder instance (for fluid interface)
      */
     public function makeAbstract() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_ABSTRACT);
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Stmt\Class_::MODIFIER_ABSTRACT);
 
         return $this;
     }
@@ -74,7 +78,13 @@ class Class_ extends Declaration
      * @return $this The builder instance (for fluid interface)
      */
     public function makeFinal() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
+
+        return $this;
+    }
+
+    public function makeReadonly() {
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Stmt\Class_::MODIFIER_READONLY);
 
         return $this;
     }
@@ -90,18 +100,31 @@ class Class_ extends Declaration
         $stmt = BuilderHelpers::normalizeNode($stmt);
 
         $targets = [
-            'Stmt_TraitUse'    => &$this->uses,
-            'Stmt_ClassConst'  => &$this->constants,
-            'Stmt_Property'    => &$this->properties,
-            'Stmt_ClassMethod' => &$this->methods,
+            Stmt\TraitUse::class    => &$this->uses,
+            Stmt\ClassConst::class  => &$this->constants,
+            Stmt\Property::class    => &$this->properties,
+            Stmt\ClassMethod::class => &$this->methods,
         ];
 
-        $type = $stmt->getType();
-        if (!isset($targets[$type])) {
-            throw new \LogicException(sprintf('Unexpected node of type "%s"', $type));
+        $class = \get_class($stmt);
+        if (!isset($targets[$class])) {
+            throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
         }
 
-        $targets[$type][] = $stmt;
+        $targets[$class][] = $stmt;
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
 
         return $this;
     }
@@ -117,6 +140,7 @@ class Class_ extends Declaration
             'extends' => $this->extends,
             'implements' => $this->implements,
             'stmts' => array_merge($this->uses, $this->constants, $this->properties, $this->methods),
+            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
-}
+}

lib/PhpParser/Builder/Declaration.php

@@ -40,4 +40,4 @@ abstract class Declaration implements PhpParser\Builder
 
         return $this;
     }
-}
+}

lib/PhpParser/Builder/EnumCase.php

@@ -0,0 +1,85 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PhpParser\Builder;
+
+use PhpParser;
+use PhpParser\BuilderHelpers;
+use PhpParser\Node;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Stmt;
+
+class EnumCase implements PhpParser\Builder
+{
+    protected $name;
+    protected $value = null;
+    protected $attributes = [];
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
+    /**
+     * Creates an enum case builder.
+     *
+     * @param string|Identifier $name  Name
+     */
+    public function __construct($name) {
+        $this->name = $name;
+    }
+
+    /**
+     * Sets the value.
+     *
+     * @param Node\Expr|string|int $value
+     *
+     * @return $this
+     */
+    public function setValue($value) {
+        $this->value = BuilderHelpers::normalizeValue($value);
+
+        return $this;
+    }
+
+    /**
+     * Sets doc comment for the constant.
+     *
+     * @param PhpParser\Comment\Doc|string $docComment Doc comment to set
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function setDocComment($docComment) {
+        $this->attributes = [
+            'comments' => [BuilderHelpers::normalizeDocComment($docComment)]
+        ];
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
+    /**
+     * Returns the built enum case node.
+     *
+     * @return Stmt\EnumCase The built constant node
+     */
+    public function getNode(): PhpParser\Node {
+        return new Stmt\EnumCase(
+            $this->name,
+            $this->value,
+            $this->attributeGroups,
+            $this->attributes
+        );
+    }
+}

lib/PhpParser/Builder/Enum_.php

@@ -0,0 +1,117 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Builder;
+
+use PhpParser;
+use PhpParser\BuilderHelpers;
+use PhpParser\Node;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Name;
+use PhpParser\Node\Stmt;
+
+class Enum_ extends Declaration
+{
+    protected $name;
+    protected $scalarType = null;
+
+    protected $implements = [];
+
+    protected $uses = [];
+    protected $enumCases = [];
+    protected $constants = [];
+    protected $methods = [];
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
+    /**
+     * Creates an enum builder.
+     *
+     * @param string $name Name of the enum
+     */
+    public function __construct(string $name) {
+        $this->name = $name;
+    }
+
+    /**
+     * Sets the scalar type.
+     *
+     * @param string|Identifier $type
+     *
+     * @return $this
+     */
+    public function setScalarType($scalarType) {
+        $this->scalarType = BuilderHelpers::normalizeType($scalarType);
+
+        return $this;
+    }
+
+    /**
+     * Implements one or more interfaces.
+     *
+     * @param Name|string ...$interfaces Names of interfaces to implement
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function implement(...$interfaces) {
+        foreach ($interfaces as $interface) {
+            $this->implements[] = BuilderHelpers::normalizeName($interface);
+        }
+
+        return $this;
+    }
+
+    /**
+     * Adds a statement.
+     *
+     * @param Stmt|PhpParser\Builder $stmt The statement to add
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addStmt($stmt) {
+        $stmt = BuilderHelpers::normalizeNode($stmt);
+
+        $targets = [
+            Stmt\TraitUse::class    => &$this->uses,
+            Stmt\EnumCase::class    => &$this->enumCases,
+            Stmt\ClassConst::class  => &$this->constants,
+            Stmt\ClassMethod::class => &$this->methods,
+        ];
+
+        $class = \get_class($stmt);
+        if (!isset($targets[$class])) {
+            throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
+        }
+
+        $targets[$class][] = $stmt;
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
+    /**
+     * Returns the built class node.
+     *
+     * @return Stmt\Enum_ The built enum node
+     */
+    public function getNode() : PhpParser\Node {
+        return new Stmt\Enum_($this->name, [
+            'scalarType' => $this->scalarType,
+            'implements' => $this->implements,
+            'stmts' => array_merge($this->uses, $this->enumCases, $this->constants, $this->methods),
+            'attrGroups' => $this->attributeGroups,
+        ], $this->attributes);
+    }
+}

lib/PhpParser/Builder/FunctionLike.php

@@ -61,8 +61,7 @@ abstract class FunctionLike extends Declaration
     /**
      * Sets the return type for PHP 7.
      *
-     * @param string|Node\Name|Node\NullableType $type One of array, callable, string, int, float,
-     *                                                 bool, iterable, or a class/interface name.
+     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type
      *
      * @return $this The builder instance (for fluid interface)
      */

lib/PhpParser/Builder/Function_.php

@@ -12,6 +12,9 @@ class Function_ extends FunctionLike
     protected $name;
     protected $stmts = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a function builder.
      *
@@ -34,6 +37,19 @@ class Function_ extends FunctionLike
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built function node.
      *
@@ -45,6 +61,7 @@ class Function_ extends FunctionLike
             'params'     => $this->params,
             'returnType' => $this->returnType,
             'stmts'      => $this->stmts,
+            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Interface_.php

@@ -4,6 +4,7 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Node;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
@@ -14,6 +15,9 @@ class Interface_ extends Declaration
     protected $constants = [];
     protected $methods = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates an interface builder.
      *
@@ -48,25 +52,32 @@ class Interface_ extends Declaration
     public function addStmt($stmt) {
         $stmt = BuilderHelpers::normalizeNode($stmt);
 
-        $type = $stmt->getType();
-        switch ($type) {
-            case 'Stmt_ClassConst':
-                $this->constants[] = $stmt;
-                break;
-
-            case 'Stmt_ClassMethod':
-                // we erase all statements in the body of an interface method
-                $stmt->stmts = null;
-                $this->methods[] = $stmt;
-                break;
-
-            default:
-                throw new \LogicException(sprintf('Unexpected node of type "%s"', $type));
+        if ($stmt instanceof Stmt\ClassConst) {
+            $this->constants[] = $stmt;
+        } elseif ($stmt instanceof Stmt\ClassMethod) {
+            // we erase all statements in the body of an interface method
+            $stmt->stmts = null;
+            $this->methods[] = $stmt;
+        } else {
+            throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
         }
 
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built interface node.
      *
@@ -76,6 +87,7 @@ class Interface_ extends Declaration
         return new Stmt\Interface_($this->name, [
             'extends' => $this->extends,
             'stmts' => array_merge($this->constants, $this->methods),
+            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
-}
+}

lib/PhpParser/Builder/Method.php

@@ -15,6 +15,9 @@ class Method extends FunctionLike
     /** @var array|null */
     protected $stmts = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a method builder.
      *
@@ -112,6 +115,19 @@ class Method extends FunctionLike
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built method node.
      *
@@ -124,6 +140,7 @@ class Method extends FunctionLike
             'params'     => $this->params,
             'returnType' => $this->returnType,
             'stmts'      => $this->stmts,
+            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Namespace_.php

@@ -7,7 +7,7 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Namespace_ implements PhpParser\Builder
+class Namespace_ extends Declaration
 {
     private $name;
     private $stmts = [];
@@ -34,27 +34,12 @@ class Namespace_ implements PhpParser\Builder
         return $this;
     }
 
-    /**
-     * Adds multiple statements.
-     *
-     * @param array $stmts The statements to add
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addStmts(array $stmts) {
-        foreach ($stmts as $stmt) {
-            $this->addStmt($stmt);
-        }
-
-        return $this;
-    }
-
     /**
      * Returns the built node.
      *
-     * @return Node The built node
+     * @return Stmt\Namespace_ The built node
      */
     public function getNode() : Node {
-        return new Stmt\Namespace_($this->name, $this->stmts);
+        return new Stmt\Namespace_($this->name, $this->stmts, $this->attributes);
     }
 }

lib/PhpParser/Builder/Param.php

@@ -12,13 +12,18 @@ class Param implements PhpParser\Builder
 
     protected $default = null;
 
-    /** @var string|Node\Name|Node\NullableType|null */
+    /** @var Node\Identifier|Node\Name|Node\NullableType|null */
     protected $type = null;
 
     protected $byRef = false;
 
     protected $variadic = false;
 
+    protected $flags = 0;
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a parameter builder.
      *
@@ -42,13 +47,13 @@ class Param implements PhpParser\Builder
     }
 
     /**
-     * Sets type hint for the parameter.
+     * Sets type for the parameter.
      *
-     * @param string|Node\Name|Node\NullableType $type Type hint to use
+     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type Parameter type
      *
      * @return $this The builder instance (for fluid interface)
      */
-    public function setTypeHint($type) {
+    public function setType($type) {
         $this->type = BuilderHelpers::normalizeType($type);
         if ($this->type == 'void') {
             throw new \LogicException('Parameter type cannot be void');
@@ -57,6 +62,19 @@ class Param implements PhpParser\Builder
         return $this;
     }
 
+    /**
+     * Sets type for the parameter.
+     *
+     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type Parameter type
+     *
+     * @return $this The builder instance (for fluid interface)
+     *
+     * @deprecated Use setType() instead
+     */
+    public function setTypeHint($type) {
+        return $this->setType($type);
+    }
+
     /**
      * Make the parameter accept the value by reference.
      *
@@ -79,6 +97,63 @@ class Param implements PhpParser\Builder
         return $this;
     }
 
+    /**
+     * Makes the (promoted) parameter public.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makePublic() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Node\Stmt\Class_::MODIFIER_PUBLIC);
+
+        return $this;
+    }
+
+    /**
+     * Makes the (promoted) parameter protected.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeProtected() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Node\Stmt\Class_::MODIFIER_PROTECTED);
+
+        return $this;
+    }
+
+    /**
+     * Makes the (promoted) parameter private.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makePrivate() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Node\Stmt\Class_::MODIFIER_PRIVATE);
+
+        return $this;
+    }
+
+    /**
+     * Makes the (promoted) parameter readonly.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeReadonly() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Node\Stmt\Class_::MODIFIER_READONLY);
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built parameter node.
      *
@@ -87,7 +162,7 @@ class Param implements PhpParser\Builder
     public function getNode() : Node {
         return new Node\Param(
             new Node\Expr\Variable($this->name),
-            $this->default, $this->type, $this->byRef, $this->variadic
+            $this->default, $this->type, $this->byRef, $this->variadic, [], $this->flags, $this->attributeGroups
         );
     }
 }

lib/PhpParser/Builder/Property.php

@@ -4,7 +4,11 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Node;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
+use PhpParser\Node\ComplexType;
 
 class Property implements PhpParser\Builder
 {
@@ -14,6 +18,12 @@ class Property implements PhpParser\Builder
     protected $default = null;
     protected $attributes = [];
 
+    /** @var null|Identifier|Name|NullableType */
+    protected $type;
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates a property builder.
      *
@@ -67,6 +77,17 @@ class Property implements PhpParser\Builder
         return $this;
     }
 
+    /**
+     * Makes the property readonly.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeReadonly() {
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_READONLY);
+
+        return $this;
+    }
+
     /**
      * Sets default value for the property.
      *
@@ -95,6 +116,32 @@ class Property implements PhpParser\Builder
         return $this;
     }
 
+    /**
+     * Sets the property type for PHP 7.4+.
+     *
+     * @param string|Name|Identifier|ComplexType $type
+     *
+     * @return $this
+     */
+    public function setType($type) {
+        $this->type = BuilderHelpers::normalizeType($type);
+
+        return $this;
+    }
+
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built class node.
      *
@@ -106,7 +153,9 @@ class Property implements PhpParser\Builder
             [
                 new Stmt\PropertyProperty($this->name, $this->default)
             ],
-            $this->attributes
+            $this->attributes,
+            $this->type,
+            $this->attributeGroups
         );
     }
-}
+}

lib/PhpParser/Builder/TraitUse.php

@@ -0,0 +1,64 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Builder;
+
+use PhpParser\Builder;
+use PhpParser\BuilderHelpers;
+use PhpParser\Node;
+use PhpParser\Node\Stmt;
+
+class TraitUse implements Builder
+{
+    protected $traits = [];
+    protected $adaptations = [];
+
+    /**
+     * Creates a trait use builder.
+     *
+     * @param Node\Name|string ...$traits Names of used traits
+     */
+    public function __construct(...$traits) {
+        foreach ($traits as $trait) {
+            $this->and($trait);
+        }
+    }
+
+    /**
+     * Adds used trait.
+     *
+     * @param Node\Name|string $trait Trait name
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function and($trait) {
+        $this->traits[] = BuilderHelpers::normalizeName($trait);
+        return $this;
+    }
+
+    /**
+     * Adds trait adaptation.
+     *
+     * @param Stmt\TraitUseAdaptation|Builder\TraitUseAdaptation $adaptation Trait adaptation
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function with($adaptation) {
+        $adaptation = BuilderHelpers::normalizeNode($adaptation);
+
+        if (!$adaptation instanceof Stmt\TraitUseAdaptation) {
+            throw new \LogicException('Adaptation must have type TraitUseAdaptation');
+        }
+
+        $this->adaptations[] = $adaptation;
+        return $this;
+    }
+
+    /**
+     * Returns the built node.
+     *
+     * @return Node The built node
+     */
+    public function getNode() : Node {
+        return new Stmt\TraitUse($this->traits, $this->adaptations);
+    }
+}

lib/PhpParser/Builder/TraitUseAdaptation.php

@@ -0,0 +1,148 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Builder;
+
+use PhpParser\Builder;
+use PhpParser\BuilderHelpers;
+use PhpParser\Node;
+use PhpParser\Node\Stmt;
+
+class TraitUseAdaptation implements Builder
+{
+    const TYPE_UNDEFINED  = 0;
+    const TYPE_ALIAS      = 1;
+    const TYPE_PRECEDENCE = 2;
+
+    /** @var int Type of building adaptation */
+    protected $type;
+
+    protected $trait;
+    protected $method;
+
+    protected $modifier = null;
+    protected $alias = null;
+
+    protected $insteadof = [];
+
+    /**
+     * Creates a trait use adaptation builder.
+     *
+     * @param Node\Name|string|null  $trait  Name of adaptated trait
+     * @param Node\Identifier|string $method Name of adaptated method
+     */
+    public function __construct($trait, $method) {
+        $this->type = self::TYPE_UNDEFINED;
+
+        $this->trait = is_null($trait)? null: BuilderHelpers::normalizeName($trait);
+        $this->method = BuilderHelpers::normalizeIdentifier($method);
+    }
+
+    /**
+     * Sets alias of method.
+     *
+     * @param Node\Identifier|string $alias Alias for adaptated method
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function as($alias) {
+        if ($this->type === self::TYPE_UNDEFINED) {
+            $this->type = self::TYPE_ALIAS;
+        }
+
+        if ($this->type !== self::TYPE_ALIAS) {
+            throw new \LogicException('Cannot set alias for not alias adaptation buider');
+        }
+
+        $this->alias = $alias;
+        return $this;
+    }
+
+    /**
+     * Sets adaptated method public.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makePublic() {
+        $this->setModifier(Stmt\Class_::MODIFIER_PUBLIC);
+        return $this;
+    }
+
+    /**
+     * Sets adaptated method protected.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeProtected() {
+        $this->setModifier(Stmt\Class_::MODIFIER_PROTECTED);
+        return $this;
+    }
+
+    /**
+     * Sets adaptated method private.
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makePrivate() {
+        $this->setModifier(Stmt\Class_::MODIFIER_PRIVATE);
+        return $this;
+    }
+
+    /**
+     * Adds overwritten traits.
+     *
+     * @param Node\Name|string ...$traits Traits for overwrite
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function insteadof(...$traits) {
+        if ($this->type === self::TYPE_UNDEFINED) {
+            if (is_null($this->trait)) {
+                throw new \LogicException('Precedence adaptation must have trait');
+            }
+
+            $this->type = self::TYPE_PRECEDENCE;
+        }
+
+        if ($this->type !== self::TYPE_PRECEDENCE) {
+            throw new \LogicException('Cannot add overwritten traits for not precedence adaptation buider');
+        }
+
+        foreach ($traits as $trait) {
+            $this->insteadof[] = BuilderHelpers::normalizeName($trait);
+        }
+
+        return $this;
+    }
+
+    protected function setModifier(int $modifier) {
+        if ($this->type === self::TYPE_UNDEFINED) {
+            $this->type = self::TYPE_ALIAS;
+        }
+
+        if ($this->type !== self::TYPE_ALIAS) {
+            throw new \LogicException('Cannot set access modifier for not alias adaptation buider');
+        }
+
+        if (is_null($this->modifier)) {
+            $this->modifier = $modifier;
+        } else {
+            throw new \LogicException('Multiple access type modifiers are not allowed');
+        }
+    }
+
+    /**
+     * Returns the built node.
+     *
+     * @return Node The built node
+     */
+    public function getNode() : Node {
+        switch ($this->type) {
+            case self::TYPE_ALIAS:
+                return new Stmt\TraitUseAdaptation\Alias($this->trait, $this->method, $this->modifier, $this->alias);
+            case self::TYPE_PRECEDENCE:
+                return new Stmt\TraitUseAdaptation\Precedence($this->trait, $this->method, $this->insteadof);
+            default:
+                throw new \LogicException('Type of adaptation is not defined');
+        }
+    }
+}

lib/PhpParser/Builder/Trait_.php

@@ -4,6 +4,7 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
 class Trait_ extends Declaration
@@ -13,6 +14,9 @@ class Trait_ extends Declaration
     protected $properties = [];
     protected $methods = [];
 
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
+
     /**
      * Creates an interface builder.
      *
@@ -34,9 +38,9 @@ class Trait_ extends Declaration
 
         if ($stmt instanceof Stmt\Property) {
             $this->properties[] = $stmt;
-        } else if ($stmt instanceof Stmt\ClassMethod) {
+        } elseif ($stmt instanceof Stmt\ClassMethod) {
             $this->methods[] = $stmt;
-        } else if ($stmt instanceof Stmt\TraitUse) {
+        } elseif ($stmt instanceof Stmt\TraitUse) {
             $this->uses[] = $stmt;
         } else {
             throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
@@ -45,6 +49,19 @@ class Trait_ extends Declaration
         return $this;
     }
 
+    /**
+     * Adds an attribute group.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function addAttribute($attribute) {
+        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
+
+        return $this;
+    }
+
     /**
      * Returns the built trait node.
      *
@@ -53,7 +70,8 @@ class Trait_ extends Declaration
     public function getNode() : PhpParser\Node {
         return new Stmt\Trait_(
             $this->name, [
-                'stmts' => array_merge($this->uses, $this->properties, $this->methods)
+                'stmts' => array_merge($this->uses, $this->properties, $this->methods),
+                'attrGroups' => $this->attributeGroups,
             ], $this->attributes
         );
     }

lib/PhpParser/Builder/Use_.php

@@ -7,7 +7,8 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Use_ implements Builder {
+class Use_ implements Builder
+{
     protected $name;
     protected $type;
     protected $alias = null;
@@ -38,7 +39,7 @@ class Use_ implements Builder {
     /**
      * Returns the built node.
      *
-     * @return Node The built node
+     * @return Stmt\Use_ The built node
      */
     public function getNode() : Node {
         return new Stmt\Use_([

lib/PhpParser/BuilderFactory.php

@@ -5,11 +5,28 @@ namespace PhpParser;
 use PhpParser\Node\Arg;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Expr\BinaryOp\Concat;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Name;
 use PhpParser\Node\Scalar\String_;
 use PhpParser\Node\Stmt\Use_;
 
 class BuilderFactory
 {
+    /**
+     * Creates an attribute node.
+     *
+     * @param string|Name $name Name of the attribute
+     * @param array       $args Attribute named arguments
+     *
+     * @return Node\Attribute
+     */
+    public function attribute($name, array $args = []) : Node\Attribute {
+        return new Node\Attribute(
+            BuilderHelpers::normalizeName($name),
+            $this->args($args)
+        );
+    }
+
     /**
      * Creates a namespace builder.
      *
@@ -54,6 +71,45 @@ class BuilderFactory
         return new Builder\Trait_($name);
     }
 
+    /**
+     * Creates an enum builder.
+     *
+     * @param string $name Name of the enum
+     *
+     * @return Builder\Enum_ The created enum builder
+     */
+    public function enum(string $name) : Builder\Enum_ {
+        return new Builder\Enum_($name);
+    }
+
+    /**
+     * Creates a trait use builder.
+     *
+     * @param Node\Name|string ...$traits Trait names
+     *
+     * @return Builder\TraitUse The create trait use builder
+     */
+    public function useTrait(...$traits) : Builder\TraitUse {
+        return new Builder\TraitUse(...$traits);
+    }
+
+    /**
+     * Creates a trait use adaptation builder.
+     *
+     * @param Node\Name|string|null  $trait  Trait name
+     * @param Node\Identifier|string $method Method name
+     *
+     * @return Builder\TraitUseAdaptation The create trait use adaptation builder
+     */
+    public function traitUseAdaptation($trait, $method = null) : Builder\TraitUseAdaptation {
+        if ($method === null) {
+            $method = $trait;
+            $trait = null;
+        }
+
+        return new Builder\TraitUseAdaptation($trait, $method);
+    }
+
     /**
      * Creates a method builder.
      *
@@ -101,14 +157,59 @@ class BuilderFactory
     /**
      * Creates a namespace/class use builder.
      *
-     * @param string|Node\Name Name to alias
+     * @param Node\Name|string $name Name of the entity (namespace or class) to alias
      *
-     * @return Builder\Use_ The create use builder
+     * @return Builder\Use_ The created use builder
      */
     public function use($name) : Builder\Use_ {
         return new Builder\Use_($name, Use_::TYPE_NORMAL);
     }
 
+    /**
+     * Creates a function use builder.
+     *
+     * @param Node\Name|string $name Name of the function to alias
+     *
+     * @return Builder\Use_ The created use function builder
+     */
+    public function useFunction($name) : Builder\Use_ {
+        return new Builder\Use_($name, Use_::TYPE_FUNCTION);
+    }
+
+    /**
+     * Creates a constant use builder.
+     *
+     * @param Node\Name|string $name Name of the const to alias
+     *
+     * @return Builder\Use_ The created use const builder
+     */
+    public function useConst($name) : Builder\Use_ {
+        return new Builder\Use_($name, Use_::TYPE_CONSTANT);
+    }
+
+    /**
+     * Creates a class constant builder.
+     *
+     * @param string|Identifier                          $name  Name
+     * @param Node\Expr|bool|null|int|float|string|array $value Value
+     *
+     * @return Builder\ClassConst The created use const builder
+     */
+    public function classConst($name, $value) : Builder\ClassConst {
+        return new Builder\ClassConst($name, $value);
+    }
+
+    /**
+     * Creates an enum case builder.
+     *
+     * @param string|Identifier $name  Name
+     *
+     * @return Builder\EnumCase The created use const builder
+     */
+    public function enumCase($name) : Builder\EnumCase {
+        return new Builder\EnumCase($name);
+    }
+
     /**
      * Creates node a for a literal value.
      *
@@ -120,6 +221,21 @@ class BuilderFactory
         return BuilderHelpers::normalizeValue($value);
     }
 
+    /**
+     * Creates variable node.
+     *
+     * @param string|Expr $name Name
+     *
+     * @return Expr\Variable
+     */
+    public function var($name) : Expr\Variable {
+        if (!\is_string($name) && !$name instanceof Expr) {
+            throw new \LogicException('Variable name must be string or Expr');
+        }
+
+        return new Expr\Variable($name);
+    }
+
     /**
      * Normalizes an argument list.
      *
@@ -131,16 +247,120 @@ class BuilderFactory
      */
     public function args(array $args) : array {
         $normalizedArgs = [];
-        foreach ($args as $arg) {
-            if ($arg instanceof Arg) {
-                $normalizedArgs[] = $arg;
-            } else {
-                $normalizedArgs[] = new Arg(BuilderHelpers::normalizeValue($arg));
+        foreach ($args as $key => $arg) {
+            if (!($arg instanceof Arg)) {
+                $arg = new Arg(BuilderHelpers::normalizeValue($arg));
             }
+            if (\is_string($key)) {
+                $arg->name = BuilderHelpers::normalizeIdentifier($key);
+            }
+            $normalizedArgs[] = $arg;
         }
         return $normalizedArgs;
     }
 
+    /**
+     * Creates a function call node.
+     *
+     * @param string|Name|Expr $name Function name
+     * @param array            $args Function arguments
+     *
+     * @return Expr\FuncCall
+     */
+    public function funcCall($name, array $args = []) : Expr\FuncCall {
+        return new Expr\FuncCall(
+            BuilderHelpers::normalizeNameOrExpr($name),
+            $this->args($args)
+        );
+    }
+
+    /**
+     * Creates a method call node.
+     *
+     * @param Expr                   $var  Variable the method is called on
+     * @param string|Identifier|Expr $name Method name
+     * @param array                  $args Method arguments
+     *
+     * @return Expr\MethodCall
+     */
+    public function methodCall(Expr $var, $name, array $args = []) : Expr\MethodCall {
+        return new Expr\MethodCall(
+            $var,
+            BuilderHelpers::normalizeIdentifierOrExpr($name),
+            $this->args($args)
+        );
+    }
+
+    /**
+     * Creates a static method call node.
+     *
+     * @param string|Name|Expr       $class Class name
+     * @param string|Identifier|Expr $name  Method name
+     * @param array                  $args  Method arguments
+     *
+     * @return Expr\StaticCall
+     */
+    public function staticCall($class, $name, array $args = []) : Expr\StaticCall {
+        return new Expr\StaticCall(
+            BuilderHelpers::normalizeNameOrExpr($class),
+            BuilderHelpers::normalizeIdentifierOrExpr($name),
+            $this->args($args)
+        );
+    }
+
+    /**
+     * Creates an object creation node.
+     *
+     * @param string|Name|Expr $class Class name
+     * @param array            $args  Constructor arguments
+     *
+     * @return Expr\New_
+     */
+    public function new($class, array $args = []) : Expr\New_ {
+        return new Expr\New_(
+            BuilderHelpers::normalizeNameOrExpr($class),
+            $this->args($args)
+        );
+    }
+
+    /**
+     * Creates a constant fetch node.
+     *
+     * @param string|Name $name Constant name
+     *
+     * @return Expr\ConstFetch
+     */
+    public function constFetch($name) : Expr\ConstFetch {
+        return new Expr\ConstFetch(BuilderHelpers::normalizeName($name));
+    }
+
+    /**
+     * Creates a property fetch node.
+     *
+     * @param Expr                   $var  Variable holding object
+     * @param string|Identifier|Expr $name Property name
+     *
+     * @return Expr\PropertyFetch
+     */
+    public function propertyFetch(Expr $var, $name) : Expr\PropertyFetch {
+        return new Expr\PropertyFetch($var, BuilderHelpers::normalizeIdentifierOrExpr($name));
+    }
+
+    /**
+     * Creates a class constant fetch node.
+     *
+     * @param string|Name|Expr $class Class name
+     * @param string|Identifier|Expr $name  Constant name
+     *
+     * @return Expr\ClassConstFetch
+     */
+    public function classConstFetch($class, $name): Expr\ClassConstFetch {
+        return new Expr\ClassConstFetch(
+            BuilderHelpers::normalizeNameOrExpr($class),
+            BuilderHelpers::normalizeIdentifierOrExpr($name)
+        );
+    }
+
     /**
      * Creates nested Concat nodes from a list of expressions.
      *
@@ -161,12 +381,16 @@ class BuilderFactory
         return $lastConcat;
     }
 
-    private function normalizeStringExpr($expr) {
+    /**
+     * @param string|Expr $expr
+     * @return Expr
+     */
+    private function normalizeStringExpr($expr) : Expr {
         if ($expr instanceof Expr) {
             return $expr;
         }
 
-        if (is_string($expr)) {
+        if (\is_string($expr)) {
             return new String_($expr);
         }
 

lib/PhpParser/BuilderHelpers.php

@@ -2,6 +2,7 @@
 
 namespace PhpParser;
 
+use PhpParser\Node\ComplexType;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
@@ -14,7 +15,8 @@ use PhpParser\Node\Stmt;
  *
  * @internal
  */
-final class BuilderHelpers {
+final class BuilderHelpers
+{
     /**
      * Normalizes a node: Converts builder objects to nodes.
      *
@@ -25,7 +27,9 @@ final class BuilderHelpers {
     public static function normalizeNode($node) : Node {
         if ($node instanceof Builder) {
             return $node->getNode();
-        } elseif ($node instanceof Node) {
+        }
+
+        if ($node instanceof Node) {
             return $node;
         }
 
@@ -55,7 +59,45 @@ final class BuilderHelpers {
     }
 
     /**
-     * Normalizes a name: Converts plain string names to PhpParser\Node\Name.
+     * Normalizes strings to Identifier.
+     *
+     * @param string|Identifier $name The identifier to normalize
+     *
+     * @return Identifier The normalized identifier
+     */
+    public static function normalizeIdentifier($name) : Identifier {
+        if ($name instanceof Identifier) {
+            return $name;
+        }
+
+        if (\is_string($name)) {
+            return new Identifier($name);
+        }
+
+        throw new \LogicException('Expected string or instance of Node\Identifier');
+    }
+
+    /**
+     * Normalizes strings to Identifier, also allowing expressions.
+     *
+     * @param string|Identifier|Expr $name The identifier to normalize
+     *
+     * @return Identifier|Expr The normalized identifier or expression
+     */
+    public static function normalizeIdentifierOrExpr($name) {
+        if ($name instanceof Identifier || $name instanceof Expr) {
+            return $name;
+        }
+
+        if (\is_string($name)) {
+            return new Identifier($name);
+        }
+
+        throw new \LogicException('Expected string or instance of Node\Identifier or Node\Expr');
+    }
+
+    /**
+     * Normalizes a name: Converts string names to Name nodes.
      *
      * @param Name|string $name The name to normalize
      *
@@ -64,21 +106,46 @@ final class BuilderHelpers {
     public static function normalizeName($name) : Name {
         if ($name instanceof Name) {
             return $name;
-        } elseif (is_string($name)) {
+        }
+
+        if (is_string($name)) {
             if (!$name) {
                 throw new \LogicException('Name cannot be empty');
             }
 
             if ($name[0] === '\\') {
                 return new Name\FullyQualified(substr($name, 1));
-            } elseif (0 === strpos($name, 'namespace\\')) {
-                return new Name\Relative(substr($name, strlen('namespace\\')));
-            } else {
-                return new Name($name);
             }
+
+            if (0 === strpos($name, 'namespace\\')) {
+                return new Name\Relative(substr($name, strlen('namespace\\')));
+            }
+
+            return new Name($name);
         }
 
-        throw new \LogicException('Name must be a string or an instance of PhpParser\Node\Name');
+        throw new \LogicException('Name must be a string or an instance of Node\Name');
+    }
+
+    /**
+     * Normalizes a name: Converts string names to Name nodes, while also allowing expressions.
+     *
+     * @param Expr|Name|string $name The name to normalize
+     *
+     * @return Name|Expr The normalized name or expression
+     */
+    public static function normalizeNameOrExpr($name) {
+        if ($name instanceof Expr) {
+            return $name;
+        }
+
+        if (!is_string($name) && !($name instanceof Name)) {
+            throw new \LogicException(
+                'Name must be a string or an instance of Node\Name or Node\Expr'
+            );
+        }
+
+        return self::normalizeName($name);
     }
 
     /**
@@ -87,16 +154,19 @@ final class BuilderHelpers {
      * In particular, builtin types become Identifiers, custom types become Names and nullables
      * are wrapped in NullableType nodes.
      *
-     * @param string|Name|Identifier|NullableType $type The type to normalize
+     * @param string|Name|Identifier|ComplexType $type The type to normalize
      *
-     * @return Name|Identifier|NullableType The normalized type
+     * @return Name|Identifier|ComplexType The normalized type
      */
     public static function normalizeType($type) {
         if (!is_string($type)) {
-            if (!$type instanceof Name && !$type instanceof Identifier
-                    && !$type instanceof NullableType) {
+            if (
+                !$type instanceof Name && !$type instanceof Identifier &&
+                !$type instanceof ComplexType
+            ) {
                 throw new \LogicException(
-                    'Type must be a string, or an instance of Name, Identifier or NullableType');
+                    'Type must be a string, or an instance of Name, Identifier or ComplexType'
+                );
             }
             return $type;
         }
@@ -108,7 +178,20 @@ final class BuilderHelpers {
         }
 
         $builtinTypes = [
-            'array', 'callable', 'string', 'int', 'float', 'bool', 'iterable', 'void', 'object'
+            'array',
+            'callable',
+            'bool',
+            'int',
+            'float',
+            'string',
+            'iterable',
+            'void',
+            'object',
+            'null',
+            'false',
+            'mixed',
+            'never',
+            'true',
         ];
 
         $lowerType = strtolower($type);
@@ -118,11 +201,14 @@ final class BuilderHelpers {
             $type = self::normalizeName($type);
         }
 
-        if ($nullable && (string) $type === 'void') {
-            throw new \LogicException('void type cannot be nullable');
+        $notNullableTypes = [
+            'void', 'mixed', 'never',
+        ];
+        if ($nullable && in_array((string) $type, $notNullableTypes)) {
+            throw new \LogicException(sprintf('%s type cannot be nullable', $type));
         }
 
-        return $nullable ? new Node\NullableType($type) : $type;
+        return $nullable ? new NullableType($type) : $type;
     }
 
     /**
@@ -136,21 +222,33 @@ final class BuilderHelpers {
     public static function normalizeValue($value) : Expr {
         if ($value instanceof Node\Expr) {
             return $value;
-        } elseif (is_null($value)) {
+        }
+
+        if (is_null($value)) {
             return new Expr\ConstFetch(
                 new Name('null')
             );
-        } elseif (is_bool($value)) {
+        }
+
+        if (is_bool($value)) {
             return new Expr\ConstFetch(
                 new Name($value ? 'true' : 'false')
             );
-        } elseif (is_int($value)) {
+        }
+
+        if (is_int($value)) {
             return new Scalar\LNumber($value);
-        } elseif (is_float($value)) {
+        }
+
+        if (is_float($value)) {
             return new Scalar\DNumber($value);
-        } elseif (is_string($value)) {
+        }
+
+        if (is_string($value)) {
             return new Scalar\String_($value);
-        } elseif (is_array($value)) {
+        }
+
+        if (is_array($value)) {
             $items = [];
             $lastKey = -1;
             foreach ($value as $itemKey => $itemValue) {
@@ -169,9 +267,9 @@ final class BuilderHelpers {
             }
 
             return new Expr\Array_($items);
-        } else {
-            throw new \LogicException('Invalid value');
         }
+
+        throw new \LogicException('Invalid value');
     }
 
     /**
@@ -184,11 +282,33 @@ final class BuilderHelpers {
     public static function normalizeDocComment($docComment) : Comment\Doc {
         if ($docComment instanceof Comment\Doc) {
             return $docComment;
-        } else if (is_string($docComment)) {
-            return new Comment\Doc($docComment);
-        } else {
-            throw new \LogicException('Doc comment must be a string or an instance of PhpParser\Comment\Doc');
         }
+
+        if (is_string($docComment)) {
+            return new Comment\Doc($docComment);
+        }
+
+        throw new \LogicException('Doc comment must be a string or an instance of PhpParser\Comment\Doc');
+    }
+
+    /**
+     * Normalizes a attribute: Converts attribute to the Attribute Group if needed.
+     *
+     * @param Node\Attribute|Node\AttributeGroup $attribute
+     *
+     * @return Node\AttributeGroup The Attribute Group
+     */
+    public static function normalizeAttribute($attribute) : Node\AttributeGroup
+    {
+        if ($attribute instanceof Node\AttributeGroup) {
+            return $attribute;
+        }
+
+        if (!($attribute instanceof Node\Attribute)) {
+            throw new \LogicException('Attribute must be an instance of PhpParser\Node\Attribute or PhpParser\Node\AttributeGroup');
+        }
+
+        return new Node\AttributeGroup([$attribute]);
     }
 
     /**
@@ -203,4 +323,13 @@ final class BuilderHelpers {
         Stmt\Class_::verifyModifier($modifiers, $modifier);
         return $modifiers | $modifier;
     }
+
+    /**
+     * Adds a modifier and returns new modifier bitmask.
+     * @return int New modifiers
+     */
+    public static function addClassModifier(int $existingModifiers, int $modifierToSet) : int {
+        Stmt\Class_::verifyClassModifier($existingModifiers, $modifierToSet);
+        return $existingModifiers | $modifierToSet;
+    }
 }

lib/PhpParser/Comment.php

@@ -5,20 +5,33 @@ namespace PhpParser;
 class Comment implements \JsonSerializable
 {
     protected $text;
-    protected $line;
-    protected $filePos;
+    protected $startLine;
+    protected $startFilePos;
+    protected $startTokenPos;
+    protected $endLine;
+    protected $endFilePos;
+    protected $endTokenPos;
 
     /**
      * Constructs a comment node.
      *
-     * @param string $text         Comment text (including comment delimiters like /*)
-     * @param int    $startLine    Line number the comment started on
-     * @param int    $startFilePos File offset the comment started on
+     * @param string $text          Comment text (including comment delimiters like /*)
+     * @param int    $startLine     Line number the comment started on
+     * @param int    $startFilePos  File offset the comment started on
+     * @param int    $startTokenPos Token offset the comment started on
      */
-    public function __construct(string $text, int $startLine = -1, int $startFilePos = -1) {
+    public function __construct(
+        string $text,
+        int $startLine = -1, int $startFilePos = -1, int $startTokenPos = -1,
+        int $endLine = -1, int $endFilePos = -1, int $endTokenPos = -1
+    ) {
         $this->text = $text;
-        $this->line = $startLine;
-        $this->filePos = $startFilePos;
+        $this->startLine = $startLine;
+        $this->startFilePos = $startFilePos;
+        $this->startTokenPos = $startTokenPos;
+        $this->endLine = $endLine;
+        $this->endFilePos = $endFilePos;
+        $this->endTokenPos = $endTokenPos;
     }
 
     /**
@@ -33,19 +46,88 @@ class Comment implements \JsonSerializable
     /**
      * Gets the line number the comment started on.
      *
-     * @return int Line number
+     * @return int Line number (or -1 if not available)
      */
-    public function getLine() : int {
-        return $this->line;
+    public function getStartLine() : int {
+        return $this->startLine;
     }
 
     /**
      * Gets the file offset the comment started on.
      *
+     * @return int File offset (or -1 if not available)
+     */
+    public function getStartFilePos() : int {
+        return $this->startFilePos;
+    }
+
+    /**
+     * Gets the token offset the comment started on.
+     *
+     * @return int Token offset (or -1 if not available)
+     */
+    public function getStartTokenPos() : int {
+        return $this->startTokenPos;
+    }
+
+    /**
+     * Gets the line number the comment ends on.
+     *
+     * @return int Line number (or -1 if not available)
+     */
+    public function getEndLine() : int {
+        return $this->endLine;
+    }
+
+    /**
+     * Gets the file offset the comment ends on.
+     *
+     * @return int File offset (or -1 if not available)
+     */
+    public function getEndFilePos() : int {
+        return $this->endFilePos;
+    }
+
+    /**
+     * Gets the token offset the comment ends on.
+     *
+     * @return int Token offset (or -1 if not available)
+     */
+    public function getEndTokenPos() : int {
+        return $this->endTokenPos;
+    }
+
+    /**
+     * Gets the line number the comment started on.
+     *
+     * @deprecated Use getStartLine() instead
+     *
+     * @return int Line number
+     */
+    public function getLine() : int {
+        return $this->startLine;
+    }
+
+    /**
+     * Gets the file offset the comment started on.
+     *
+     * @deprecated Use getStartFilePos() instead
+     *
      * @return int File offset
      */
     public function getFilePos() : int {
-        return $this->filePos;
+        return $this->startFilePos;
+    }
+
+    /**
+     * Gets the token offset the comment started on.
+     *
+     * @deprecated Use getStartTokenPos() instead
+     *
+     * @return int Token offset
+     */
+    public function getTokenPos() : int {
+        return $this->startTokenPos;
     }
 
     /**
@@ -124,7 +206,7 @@ class Comment implements \JsonSerializable
      */
     private function getShortestWhitespacePrefixLen(string $str) : int {
         $lines = explode("\n", $str);
-        $shortestPrefixLen = INF;
+        $shortestPrefixLen = \INF;
         foreach ($lines as $line) {
             preg_match('(^\s*)', $line, $matches);
             $prefixLen = strlen($matches[0]);
@@ -145,8 +227,13 @@ class Comment implements \JsonSerializable
         return [
             'nodeType' => $type,
             'text' => $this->text,
-            'line' => $this->line,
-            'filePos' => $this->filePos,
+            // TODO: Rename these to include "start".
+            'line' => $this->startLine,
+            'filePos' => $this->startFilePos,
+            'tokenPos' => $this->startTokenPos,
+            'endLine' => $this->endLine,
+            'endFilePos' => $this->endFilePos,
+            'endTokenPos' => $this->endTokenPos,
         ];
     }
-}
+}

lib/PhpParser/Comment/Doc.php

@@ -4,4 +4,4 @@ namespace PhpParser\Comment;
 
 class Doc extends \PhpParser\Comment
 {
-}
+}

lib/PhpParser/ConstExprEvaluationException.php

@@ -2,4 +2,5 @@
 
 namespace PhpParser;
 
-class ConstExprEvaluationException extends \Exception {}
+class ConstExprEvaluationException extends \Exception
+{}

lib/PhpParser/ConstExprEvaluator.php

@@ -2,6 +2,7 @@
 
 namespace PhpParser;
 
+use function array_merge;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Scalar;
 
@@ -20,15 +21,12 @@ use PhpParser\Node\Scalar;
  *
  * The fallback evaluator should throw ConstExprEvaluationException for nodes it cannot evaluate.
  *
- * The evaluation is performed as PHP would perform it, and as such may generate notices, warnings
- * or Errors. For example, if the expression `1%0` is evaluated, an ArithmeticError is thrown. It is
- * left to the consumer to handle these as appropriate.
- *
- * The evaluation is also dependent on runtime configuration in two respects: Firstly, floating
+ * The evaluation is dependent on runtime configuration in two respects: Firstly, floating
  * point to string conversions are affected by the precision ini setting. Secondly, they are also
  * affected by the LC_NUMERIC locale.
  */
-class ConstExprEvaluator {
+class ConstExprEvaluator
+{
     private $fallbackEvaluator;
 
     /**
@@ -39,7 +37,7 @@ class ConstExprEvaluator {
      *
      * @param callable|null $fallbackEvaluator To call if subexpression cannot be evaluated
      */
-    public function __construct(callable $fallbackEvaluator = null) {
+    public function __construct(?callable $fallbackEvaluator = null) {
         $this->fallbackEvaluator = $fallbackEvaluator ?? function(Expr $expr) {
             throw new ConstExprEvaluationException(
                 "Expression of type {$expr->getType()} cannot be evaluated"
@@ -48,7 +46,10 @@ class ConstExprEvaluator {
     }
 
     /**
-     * Evaluates a constant expression into a PHP value.
+     * Silently evaluates a constant expression into a PHP value.
+     *
+     * Thrown Errors, warnings or notices will be converted into a ConstExprEvaluationException.
+     * The original source of the exception is available through getPrevious().
      *
      * If some part of the expression cannot be evaluated, the fallback evaluator passed to the
      * constructor will be invoked. By default, if no fallback is provided, an exception of type
@@ -58,9 +59,49 @@ class ConstExprEvaluator {
      *
      * @param Expr $expr Constant expression to evaluate
      * @return mixed Result of evaluation
+     *
+     * @throws ConstExprEvaluationException if the expression cannot be evaluated or an error occurred
+     */
+    public function evaluateSilently(Expr $expr) {
+        set_error_handler(function($num, $str, $file, $line) {
+            throw new \ErrorException($str, 0, $num, $file, $line);
+        });
+
+        try {
+            return $this->evaluate($expr);
+        } catch (\Throwable $e) {
+            if (!$e instanceof ConstExprEvaluationException) {
+                $e = new ConstExprEvaluationException(
+                    "An error occurred during constant expression evaluation", 0, $e);
+            }
+            throw $e;
+        } finally {
+            restore_error_handler();
+        }
+    }
+
+    /**
+     * Directly evaluates a constant expression into a PHP value.
+     *
+     * May generate Error exceptions, warnings or notices. Use evaluateSilently() to convert these
+     * into a ConstExprEvaluationException.
+     *
+     * If some part of the expression cannot be evaluated, the fallback evaluator passed to the
+     * constructor will be invoked. By default, if no fallback is provided, an exception of type
+     * ConstExprEvaluationException is thrown.
+     *
+     * See class doc comment for caveats and limitations.
+     *
+     * @param Expr $expr Constant expression to evaluate
+     * @return mixed Result of evaluation
+     *
      * @throws ConstExprEvaluationException if the expression cannot be evaluated
      */
-    public function evaluate(Expr $expr) {
+    public function evaluateDirectly(Expr $expr) {
+        return $this->evaluate($expr);
+    }
+
+    private function evaluate(Expr $expr) {
         if ($expr instanceof Scalar\LNumber
             || $expr instanceof Scalar\DNumber
             || $expr instanceof Scalar\String_
@@ -110,6 +151,8 @@ class ConstExprEvaluator {
         foreach ($expr->items as $item) {
             if (null !== $item->key) {
                 $array[$this->evaluate($item->key)] = $this->evaluate($item->value);
+            } elseif ($item->unpack) {
+                $array = array_merge($array, $this->evaluate($item->value));
             } else {
                 $array[] = $this->evaluate($item->value);
             }
@@ -183,4 +226,4 @@ class ConstExprEvaluator {
 
         return ($this->fallbackEvaluator)($expr);
     }
-}
+}

lib/PhpParser/Error.php

@@ -15,7 +15,7 @@ class Error extends \RuntimeException
      *                              (or start line of error -- deprecated)
      */
     public function __construct(string $message, $attributes = []) {
-        $this->rawMessage = (string) $message;
+        $this->rawMessage = $message;
         if (is_array($attributes)) {
             $this->attributes = $attributes;
         } else {
@@ -51,7 +51,6 @@ class Error extends \RuntimeException
         return $this->attributes['endLine'] ?? -1;
     }
 
-
     /**
      * Gets the attributes of the node/token the error occurred at.
      *
@@ -62,7 +61,7 @@ class Error extends \RuntimeException
     }
 
     /**
-     * Sets the attributes of the node/token the error occured at.
+     * Sets the attributes of the node/token the error occurred at.
      *
      * @param array $attributes
      */
@@ -77,7 +76,7 @@ class Error extends \RuntimeException
      * @param string $message Error message
      */
     public function setRawMessage(string $message) {
-        $this->rawMessage = (string) $message;
+        $this->rawMessage = $message;
         $this->updateMessage();
     }
 
@@ -87,7 +86,7 @@ class Error extends \RuntimeException
      * @param int $line Error start line
      */
     public function setStartLine(int $line) {
-        $this->attributes['startLine'] = (int) $line;
+        $this->attributes['startLine'] = $line;
         $this->updateMessage();
     }
 
@@ -99,7 +98,7 @@ class Error extends \RuntimeException
      * @return bool
      */
     public function hasColumnInfo() : bool {
-        return isset($this->attributes['startFilePos']) && isset($this->attributes['endFilePos']);
+        return isset($this->attributes['startFilePos'], $this->attributes['endFilePos']);
     }
 
     /**

lib/PhpParser/ErrorHandler.php

@@ -10,4 +10,4 @@ interface ErrorHandler
      * @param Error $error The error that needs to be handled
      */
     public function handleError(Error $error);
-}
+}

lib/PhpParser/ErrorHandler/Collecting.php

@@ -43,4 +43,4 @@ class Collecting implements ErrorHandler
     public function clearErrors() {
         $this->errors = [];
     }
-}
+}

lib/PhpParser/ErrorHandler/Throwing.php

@@ -15,4 +15,4 @@ class Throwing implements ErrorHandler
     public function handleError(Error $error) {
         throw $error;
     }
-}
+}

lib/PhpParser/Internal/DiffElem.php

@@ -1,11 +1,12 @@
-<?php
+<?php declare(strict_types=1);
 
 namespace PhpParser\Internal;
 
 /**
  * @internal
  */
-class DiffElem {
+class DiffElem
+{
     const TYPE_KEEP = 0;
     const TYPE_REMOVE = 1;
     const TYPE_ADD = 2;
@@ -23,4 +24,4 @@ class DiffElem {
         $this->old = $old;
         $this->new = $new;
     }
-}
+}

lib/PhpParser/Internal/Differ.php

@@ -1,4 +1,4 @@
-<?php
+<?php declare(strict_types=1);
 
 namespace PhpParser\Internal;
 
@@ -10,13 +10,14 @@ namespace PhpParser\Internal;
  *
  * @internal
  */
-class Differ {
+class Differ
+{
     private $isEqual;
 
     /**
      * Create differ over the given equality relation.
      *
-     * @param callable $isEqual Equality relation with signature function($a, $b): bool
+     * @param callable $isEqual Equality relation with signature function($a, $b) : bool
      */
     public function __construct(callable $isEqual) {
         $this->isEqual = $isEqual;
@@ -160,4 +161,4 @@ class Differ {
         }
         return $newDiff;
     }
-}
+}

lib/PhpParser/Internal/PrintableNewAnonClassNode.php

@@ -0,0 +1,64 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Internal;
+
+use PhpParser\Node;
+use PhpParser\Node\Expr;
+
+/**
+ * This node is used internally by the format-preserving pretty printer to print anonymous classes.
+ *
+ * The normal anonymous class structure violates assumptions about the order of token offsets.
+ * Namely, the constructor arguments are part of the Expr\New_ node and follow the class node, even
+ * though they are actually interleaved with them. This special node type is used temporarily to
+ * restore a sane token offset order.
+ *
+ * @internal
+ */
+class PrintableNewAnonClassNode extends Expr
+{
+    /** @var Node\AttributeGroup[] PHP attribute groups */
+    public $attrGroups;
+    /** @var int Modifiers */
+    public $flags;
+    /** @var Node\Arg[] Arguments */
+    public $args;
+    /** @var null|Node\Name Name of extended class */
+    public $extends;
+    /** @var Node\Name[] Names of implemented interfaces */
+    public $implements;
+    /** @var Node\Stmt[] Statements */
+    public $stmts;
+
+    public function __construct(
+        array $attrGroups, int $flags, array $args, ?Node\Name $extends, array $implements,
+        array $stmts, array $attributes
+    ) {
+        parent::__construct($attributes);
+        $this->attrGroups = $attrGroups;
+        $this->flags = $flags;
+        $this->args = $args;
+        $this->extends = $extends;
+        $this->implements = $implements;
+        $this->stmts = $stmts;
+    }
+
+    public static function fromNewNode(Expr\New_ $newNode) {
+        $class = $newNode->class;
+        assert($class instanceof Node\Stmt\Class_);
+        // We don't assert that $class->name is null here, to allow consumers to assign unique names
+        // to anonymous classes for their own purposes. We simplify ignore the name here.
+        return new self(
+            $class->attrGroups, $class->flags, $newNode->args, $class->extends, $class->implements,
+            $class->stmts, $newNode->getAttributes()
+        );
+    }
+
+    public function getType() : string {
+        return 'Expr_PrintableNewAnonClass';
+    }
+
+    public function getSubNodeNames() : array {
+        return ['attrGroups', 'flags', 'args', 'extends', 'implements', 'stmts'];
+    }
+}

lib/PhpParser/Internal/TokenStream.php

@@ -1,15 +1,18 @@
 <?php declare(strict_types=1);
 
-namespace PhpParser;
+namespace PhpParser\Internal;
 
 /**
  * Provides operations on token streams, for use by pretty printer.
  *
  * @internal
  */
-class TokenStream {
+class TokenStream
+{
     /** @var array Tokens (in token_get_all format) */
     private $tokens;
+    /** @var int[] Map from position to indentation */
+    private $indentMap;
 
     /**
      * Create token stream instance.
@@ -18,6 +21,7 @@ class TokenStream {
      */
     public function __construct(array $tokens) {
         $this->tokens = $tokens;
+        $this->indentMap = $this->calcIndentMap();
     }
 
     /**
@@ -29,7 +33,7 @@ class TokenStream {
      * @return bool
      */
     public function haveParens(int $startPos, int $endPos) : bool {
-        return $this->haveTokenImmediativelyBefore($startPos, '(')
+        return $this->haveTokenImmediatelyBefore($startPos, '(')
             && $this->haveTokenImmediatelyAfter($endPos, ')');
     }
 
@@ -42,7 +46,8 @@ class TokenStream {
      * @return bool
      */
     public function haveBraces(int $startPos, int $endPos) : bool {
-        return $this->haveTokenImmediativelyBefore($startPos, '{')
+        return ($this->haveTokenImmediatelyBefore($startPos, '{')
+                || $this->haveTokenImmediatelyBefore($startPos, T_CURLY_OPEN))
             && $this->haveTokenImmediatelyAfter($endPos, '}');
     }
 
@@ -56,7 +61,7 @@ class TokenStream {
      *
      * @return bool Whether the expected token was found
      */
-    public function haveTokenImmediativelyBefore(int $pos, $expectedTokenType) : bool {
+    public function haveTokenImmediatelyBefore(int $pos, $expectedTokenType) : bool {
         $tokens = $this->tokens;
         $pos--;
         for (; $pos >= 0; $pos--) {
@@ -64,8 +69,8 @@ class TokenStream {
             if ($tokenType === $expectedTokenType) {
                 return true;
             }
-            if ($tokenType !== T_WHITESPACE
-                && $tokenType !== T_COMMENT && $tokenType !== T_DOC_COMMENT) {
+            if ($tokenType !== \T_WHITESPACE
+                && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
                 break;
             }
         }
@@ -90,8 +95,8 @@ class TokenStream {
             if ($tokenType === $expectedTokenType) {
                 return true;
             }
-            if ($tokenType !== T_WHITESPACE
-                && $tokenType !== T_COMMENT && $tokenType !== T_DOC_COMMENT) {
+            if ($tokenType !== \T_WHITESPACE
+                && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
                 break;
             }
         }
@@ -102,7 +107,7 @@ class TokenStream {
         $tokens = $this->tokens;
 
         $pos = $this->skipLeftWhitespace($pos);
-        if ($skipTokenType === T_WHITESPACE) {
+        if ($skipTokenType === \T_WHITESPACE) {
             return $pos;
         }
 
@@ -119,7 +124,7 @@ class TokenStream {
         $tokens = $this->tokens;
 
         $pos = $this->skipRightWhitespace($pos);
-        if ($skipTokenType === T_WHITESPACE) {
+        if ($skipTokenType === \T_WHITESPACE) {
             return $pos;
         }
 
@@ -142,7 +147,7 @@ class TokenStream {
         $tokens = $this->tokens;
         for (; $pos >= 0; $pos--) {
             $type = $tokens[$pos][0];
-            if ($type !== T_WHITESPACE && $type !== T_COMMENT && $type !== T_DOC_COMMENT) {
+            if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
         }
@@ -159,14 +164,14 @@ class TokenStream {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
             $type = $tokens[$pos][0];
-            if ($type !== T_WHITESPACE && $type !== T_COMMENT && $type !== T_DOC_COMMENT) {
+            if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
         }
         return $pos;
     }
 
-    public function findRight($pos, $findTokenType) {
+    public function findRight(int $pos, $findTokenType) {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
             $type = $tokens[$pos][0];
@@ -177,6 +182,35 @@ class TokenStream {
         return -1;
     }
 
+    /**
+     * Whether the given position range contains a certain token type.
+     *
+     * @param int $startPos Starting position (inclusive)
+     * @param int $endPos Ending position (exclusive)
+     * @param int|string $tokenType Token type to look for
+     * @return bool Whether the token occurs in the given range
+     */
+    public function haveTokenInRange(int $startPos, int $endPos, $tokenType) {
+        $tokens = $this->tokens;
+        for ($pos = $startPos; $pos < $endPos; $pos++) {
+            if ($tokens[$pos][0] === $tokenType) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public function haveBracesInRange(int $startPos, int $endPos) {
+        return $this->haveTokenInRange($startPos, $endPos, '{')
+            || $this->haveTokenInRange($startPos, $endPos, T_CURLY_OPEN)
+            || $this->haveTokenInRange($startPos, $endPos, '}');
+    }
+
+    public function haveTagInRange(int $startPos, int $endPos): bool {
+        return $this->haveTokenInRange($startPos, $endPos, \T_OPEN_TAG)
+            || $this->haveTokenInRange($startPos, $endPos, \T_CLOSE_TAG);
+    }
+
     /**
      * Get indentation before token position.
      *
@@ -185,24 +219,7 @@ class TokenStream {
      * @return int Indentation depth (in spaces)
      */
     public function getIndentationBefore(int $pos) : int {
-        $tokens = $this->tokens;
-        $indent = 0;
-        $pos--;
-        for (; $pos >= 0; $pos--) {
-            if ($tokens[$pos][0] !== T_WHITESPACE) {
-                $indent = 0;
-                continue;
-            }
-            $content = $tokens[$pos][1];
-            $newlinePos = \strrpos($content, "\n");
-            if (false !== $newlinePos) {
-                $indent += \strlen($content) - $newlinePos - 1;
-                return $indent;
-            }
-
-            $indent += \strlen($content);
-        }
-        return $indent;
+        return $this->indentMap[$pos];
     }
 
     /**
@@ -222,13 +239,13 @@ class TokenStream {
             if (\is_array($token)) {
                 $type = $token[0];
                 $content = $token[1];
-                if ($type === T_CONSTANT_ENCAPSED_STRING || $type === T_ENCAPSED_AND_WHITESPACE) {
+                if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
                     $result .= $content;
                 } else {
                     // TODO Handle non-space indentation
                     if ($indent < 0) {
                         $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
-                    } else if ($indent > 0) {
+                    } elseif ($indent > 0) {
                         $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
                     } else {
                         $result .= $content;
@@ -240,4 +257,30 @@ class TokenStream {
         }
         return $result;
     }
-}
+
+    /**
+     * Precalculate the indentation at every token position.
+     *
+     * @return int[] Token position to indentation map
+     */
+    private function calcIndentMap() {
+        $indentMap = [];
+        $indent = 0;
+        foreach ($this->tokens as $token) {
+            $indentMap[] = $indent;
+
+            if ($token[0] === \T_WHITESPACE) {
+                $content = $token[1];
+                $newlinePos = \strrpos($content, "\n");
+                if (false !== $newlinePos) {
+                    $indent = \strlen($content) - $newlinePos - 1;
+                }
+            }
+        }
+
+        // Add a sentinel for one past end of the file
+        $indentMap[] = $indent;
+
+        return $indentMap;
+    }
+}

lib/PhpParser/JsonDecoder.php

@@ -2,7 +2,8 @@
 
 namespace PhpParser;
 
-class JsonDecoder {
+class JsonDecoder
+{
     /** @var \ReflectionClass[] Node type to reflection class map */
     private $reflectionClassCache;
 
@@ -71,7 +72,11 @@ class JsonDecoder {
             throw new \RuntimeException('Comment must have text');
         }
 
-        return new $className($value['text'], $value['line'] ?? -1, $value['filePos'] ?? -1);
+        return new $className(
+            $value['text'],
+            $value['line'] ?? -1, $value['filePos'] ?? -1, $value['tokenPos'] ?? -1,
+            $value['endLine'] ?? -1, $value['endFilePos'] ?? -1, $value['endTokenPos'] ?? -1
+        );
     }
 
     private function reflectionClassFromNodeType(string $nodeType) : \ReflectionClass {
@@ -95,4 +100,4 @@ class JsonDecoder {
 
         throw new \RuntimeException("Unknown node type \"$nodeType\"");
     }
-}
+}

lib/PhpParser/Lexer.php

@@ -15,8 +15,15 @@ class Lexer
 
     protected $tokenMap;
     protected $dropTokens;
+    protected $identifierTokens;
 
-    protected $usedAttributes;
+    private $attributeStartLineUsed;
+    private $attributeEndLineUsed;
+    private $attributeStartTokenPosUsed;
+    private $attributeEndTokenPosUsed;
+    private $attributeStartFilePosUsed;
+    private $attributeEndFilePosUsed;
+    private $attributeCommentsUsed;
 
     /**
      * Creates a Lexer.
@@ -28,21 +35,28 @@ class Lexer
      *                       first three. For more info see getNextToken() docs.
      */
     public function __construct(array $options = []) {
-        // map from internal tokens to PhpParser tokens
+        // Create Map from internal tokens to PhpParser tokens.
+        $this->defineCompatibilityTokens();
         $this->tokenMap = $this->createTokenMap();
+        $this->identifierTokens = $this->createIdentifierTokenMap();
 
         // map of tokens to drop while lexing (the map is only used for isset lookup,
         // that's why the value is simply set to 1; the value is never actually used.)
         $this->dropTokens = array_fill_keys(
-            [T_WHITESPACE, T_OPEN_TAG, T_COMMENT, T_DOC_COMMENT], 1
+            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
         );
 
-        // the usedAttributes member is a map of the used attribute names to a dummy
-        // value (here "true")
-        $options += [
-            'usedAttributes' => ['comments', 'startLine', 'endLine'],
-        ];
-        $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
+        $defaultAttributes = ['comments', 'startLine', 'endLine'];
+        $usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
+
+        // Create individual boolean properties to make these checks faster.
+        $this->attributeStartLineUsed = isset($usedAttributes['startLine']);
+        $this->attributeEndLineUsed = isset($usedAttributes['endLine']);
+        $this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
+        $this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
+        $this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
+        $this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
+        $this->attributeCommentsUsed = isset($usedAttributes['comments']);
     }
 
     /**
@@ -55,7 +69,7 @@ class Lexer
      * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
      *                                        ErrorHandler\Throwing
      */
-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+    public function startLexing(string $code, ?ErrorHandler $errorHandler = null) {
         if (null === $errorHandler) {
             $errorHandler = new ErrorHandler\Throwing();
         }
@@ -71,9 +85,8 @@ class Lexer
 
         $scream = ini_set('xdebug.scream', '0');
 
-        error_clear_last();
         $this->tokens = @token_get_all($code);
-        $this->handleErrors($errorHandler);
+        $this->postprocessTokens($errorHandler);
 
         if (false !== $scream) {
             ini_set('xdebug.scream', $scream);
@@ -81,13 +94,9 @@ class Lexer
     }
 
     private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
+        $tokens = [];
         for ($i = $start; $i < $end; $i++) {
             $chr = $this->code[$i];
-            if ($chr === 'b' || $chr === 'B') {
-                // HHVM does not treat b" tokens correctly, so ignore these
-                continue;
-            }
-
             if ($chr === "\0") {
                 // PHP cuts error message after null byte, so need special case
                 $errorMsg = 'Unexpected null byte';
@@ -97,6 +106,7 @@ class Lexer
                 );
             }
 
+            $tokens[] = [\T_BAD_CHARACTER, $chr, $line];
             $errorHandler->handleError(new Error($errorMsg, [
                 'startLine' => $line,
                 'endLine' => $line,
@@ -104,6 +114,7 @@ class Lexer
                 'endFilePos' => $i,
             ]));
         }
+        return $tokens;
     }
 
     /**
@@ -112,48 +123,121 @@ class Lexer
      * @return bool
      */
     private function isUnterminatedComment($token) : bool {
-        return ($token[0] === T_COMMENT || $token[0] === T_DOC_COMMENT)
+        return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
             && substr($token[1], 0, 2) === '/*'
             && substr($token[1], -2) !== '*/';
     }
 
-    /**
-     * Check whether an error *may* have occurred during tokenization.
-     *
-     * @return bool
-     */
-    private function errorMayHaveOccurred() : bool {
-        if (defined('HHVM_VERSION')) {
-            // In HHVM token_get_all() does not throw warnings, so we need to conservatively
-            // assume that an error occurred
-            return true;
-        }
-
-        return null !== error_get_last();
-    }
-
-    protected function handleErrors(ErrorHandler $errorHandler) {
-        if (!$this->errorMayHaveOccurred()) {
-            return;
-        }
-
+    protected function postprocessTokens(ErrorHandler $errorHandler) {
         // PHP's error handling for token_get_all() is rather bad, so if we want detailed
         // error information we need to compute it ourselves. Invalid character errors are
         // detected by finding "gaps" in the token array. Unterminated comments are detected
         // by checking if a trailing comment has a "*/" at the end.
+        //
+        // Additionally, we perform a number of canonicalizations here:
+        //  * Use the PHP 8.0 comment format, which does not include trailing whitespace anymore.
+        //  * Use PHP 8.0 T_NAME_* tokens.
+        //  * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
+        //    T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
 
         $filePos = 0;
         $line = 1;
-        foreach ($this->tokens as $token) {
+        $numTokens = \count($this->tokens);
+        for ($i = 0; $i < $numTokens; $i++) {
+            $token = $this->tokens[$i];
+
+            // Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
+            // In this case we only need to emit an error.
+            if ($token[0] === \T_BAD_CHARACTER) {
+                $this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
+            }
+
+            if ($token[0] === \T_COMMENT && substr($token[1], 0, 2) !== '/*'
+                    && preg_match('/(\r\n|\n|\r)$/D', $token[1], $matches)) {
+                $trailingNewline = $matches[0];
+                $token[1] = substr($token[1], 0, -strlen($trailingNewline));
+                $this->tokens[$i] = $token;
+                if (isset($this->tokens[$i + 1]) && $this->tokens[$i + 1][0] === \T_WHITESPACE) {
+                    // Move trailing newline into following T_WHITESPACE token, if it already exists.
+                    $this->tokens[$i + 1][1] = $trailingNewline . $this->tokens[$i + 1][1];
+                    $this->tokens[$i + 1][2]--;
+                } else {
+                    // Otherwise, we need to create a new T_WHITESPACE token.
+                    array_splice($this->tokens, $i + 1, 0, [
+                        [\T_WHITESPACE, $trailingNewline, $line],
+                    ]);
+                    $numTokens++;
+                }
+            }
+
+            // Emulate PHP 8 T_NAME_* tokens, by combining sequences of T_NS_SEPARATOR and T_STRING
+            // into a single token.
+            if (\is_array($token)
+                    && ($token[0] === \T_NS_SEPARATOR || isset($this->identifierTokens[$token[0]]))) {
+                $lastWasSeparator = $token[0] === \T_NS_SEPARATOR;
+                $text = $token[1];
+                for ($j = $i + 1; isset($this->tokens[$j]); $j++) {
+                    if ($lastWasSeparator) {
+                        if (!isset($this->identifierTokens[$this->tokens[$j][0]])) {
+                            break;
+                        }
+                        $lastWasSeparator = false;
+                    } else {
+                        if ($this->tokens[$j][0] !== \T_NS_SEPARATOR) {
+                            break;
+                        }
+                        $lastWasSeparator = true;
+                    }
+                    $text .= $this->tokens[$j][1];
+                }
+                if ($lastWasSeparator) {
+                    // Trailing separator is not part of the name.
+                    $j--;
+                    $text = substr($text, 0, -1);
+                }
+                if ($j > $i + 1) {
+                    if ($token[0] === \T_NS_SEPARATOR) {
+                        $type = \T_NAME_FULLY_QUALIFIED;
+                    } else if ($token[0] === \T_NAMESPACE) {
+                        $type = \T_NAME_RELATIVE;
+                    } else {
+                        $type = \T_NAME_QUALIFIED;
+                    }
+                    $token = [$type, $text, $line];
+                    array_splice($this->tokens, $i, $j - $i, [$token]);
+                    $numTokens -= $j - $i - 1;
+                }
+            }
+
+            if ($token === '&') {
+                $next = $i + 1;
+                while (isset($this->tokens[$next]) && $this->tokens[$next][0] === \T_WHITESPACE) {
+                    $next++;
+                }
+                $followedByVarOrVarArg = isset($this->tokens[$next]) &&
+                    ($this->tokens[$next][0] === \T_VARIABLE || $this->tokens[$next][0] === \T_ELLIPSIS);
+                $this->tokens[$i] = $token = [
+                    $followedByVarOrVarArg
+                        ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
+                        : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG,
+                    '&',
+                    $line,
+                ];
+            }
+
             $tokenValue = \is_string($token) ? $token : $token[1];
             $tokenLen = \strlen($tokenValue);
 
             if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
                 // Something is missing, must be an invalid character
                 $nextFilePos = strpos($this->code, $tokenValue, $filePos);
-                $this->handleInvalidCharacterRange(
+                $badCharTokens = $this->handleInvalidCharacterRange(
                     $filePos, $nextFilePos, $line, $errorHandler);
-                $filePos = (int)$nextFilePos;
+                $filePos = (int) $nextFilePos;
+
+                array_splice($this->tokens, $i, 0, $badCharTokens);
+                $numTokens += \count($badCharTokens);
+                $i += \count($badCharTokens);
             }
 
             $filePos += $tokenLen;
@@ -173,11 +257,12 @@ class Lexer
 
                 // Emulate the PHP behavior
                 $isDocComment = isset($comment[3]) && $comment[3] === '*';
-                $this->tokens[] = [$isDocComment ? T_DOC_COMMENT : T_COMMENT, $comment, $line];
+                $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
             } else {
                 // Invalid characters at the end of the input
-                $this->handleInvalidCharacterRange(
+                $badCharTokens = $this->handleInvalidCharacterRange(
                     $filePos, \strlen($this->code), $line, $errorHandler);
+                $this->tokens = array_merge($this->tokens, $badCharTokens);
             }
             return;
         }
@@ -230,13 +315,13 @@ class Lexer
                 $token = "\0";
             }
 
-            if (isset($this->usedAttributes['startLine'])) {
+            if ($this->attributeStartLineUsed) {
                 $startAttributes['startLine'] = $this->line;
             }
-            if (isset($this->usedAttributes['startTokenPos'])) {
+            if ($this->attributeStartTokenPosUsed) {
                 $startAttributes['startTokenPos'] = $this->pos;
             }
-            if (isset($this->usedAttributes['startFilePos'])) {
+            if ($this->attributeStartFilePosUsed) {
                 $startAttributes['startFilePos'] = $this->filePos;
             }
 
@@ -253,36 +338,43 @@ class Lexer
             } elseif (!isset($this->dropTokens[$token[0]])) {
                 $value = $token[1];
                 $id = $this->tokenMap[$token[0]];
-                if (T_CLOSE_TAG === $token[0]) {
-                    $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
-                } else if (T_INLINE_HTML === $token[0]) {
+                if (\T_CLOSE_TAG === $token[0]) {
+                    $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n")
+                        || false !== strpos($token[1], "\r");
+                } elseif (\T_INLINE_HTML === $token[0]) {
                     $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
                 }
 
                 $this->line += substr_count($value, "\n");
                 $this->filePos += \strlen($value);
             } else {
-                if (T_COMMENT === $token[0] || T_DOC_COMMENT === $token[0]) {
-                    if (isset($this->usedAttributes['comments'])) {
-                        $comment = T_DOC_COMMENT === $token[0]
-                            ? new Comment\Doc($token[1], $this->line, $this->filePos)
-                            : new Comment($token[1], $this->line, $this->filePos);
+                $origLine = $this->line;
+                $origFilePos = $this->filePos;
+                $this->line += substr_count($token[1], "\n");
+                $this->filePos += \strlen($token[1]);
+
+                if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
+                    if ($this->attributeCommentsUsed) {
+                        $comment = \T_DOC_COMMENT === $token[0]
+                            ? new Comment\Doc($token[1],
+                                $origLine, $origFilePos, $this->pos,
+                                $this->line, $this->filePos - 1, $this->pos)
+                            : new Comment($token[1],
+                                $origLine, $origFilePos, $this->pos,
+                                $this->line, $this->filePos - 1, $this->pos);
                         $startAttributes['comments'][] = $comment;
                     }
                 }
-
-                $this->line += substr_count($token[1], "\n");
-                $this->filePos += \strlen($token[1]);
                 continue;
             }
 
-            if (isset($this->usedAttributes['endLine'])) {
+            if ($this->attributeEndLineUsed) {
                 $endAttributes['endLine'] = $this->line;
             }
-            if (isset($this->usedAttributes['endTokenPos'])) {
+            if ($this->attributeEndTokenPosUsed) {
                 $endAttributes['endTokenPos'] = $this->pos;
             }
-            if (isset($this->usedAttributes['endFilePos'])) {
+            if ($this->attributeEndFilePosUsed) {
                 $endAttributes['endFilePos'] = $this->filePos - 1;
             }
 
@@ -329,6 +421,66 @@ class Lexer
         return substr($textAfter, strlen($matches[0]));
     }
 
+    private function defineCompatibilityTokens() {
+        static $compatTokensDefined = false;
+        if ($compatTokensDefined) {
+            return;
+        }
+
+        $compatTokens = [
+            // PHP 7.4
+            'T_BAD_CHARACTER',
+            'T_FN',
+            'T_COALESCE_EQUAL',
+            // PHP 8.0
+            'T_NAME_QUALIFIED',
+            'T_NAME_FULLY_QUALIFIED',
+            'T_NAME_RELATIVE',
+            'T_MATCH',
+            'T_NULLSAFE_OBJECT_OPERATOR',
+            'T_ATTRIBUTE',
+            // PHP 8.1
+            'T_ENUM',
+            'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_READONLY',
+        ];
+
+        // PHP-Parser might be used together with another library that also emulates some or all
+        // of these tokens. Perform a sanity-check that all already defined tokens have been
+        // assigned a unique ID.
+        $usedTokenIds = [];
+        foreach ($compatTokens as $token) {
+            if (\defined($token)) {
+                $tokenId = \constant($token);
+                $clashingToken = $usedTokenIds[$tokenId] ?? null;
+                if ($clashingToken !== null) {
+                    throw new \Error(sprintf(
+                        'Token %s has same ID as token %s, ' .
+                        'you may be using a library with broken token emulation',
+                        $token, $clashingToken
+                    ));
+                }
+                $usedTokenIds[$tokenId] = $token;
+            }
+        }
+
+        // Now define any tokens that have not yet been emulated. Try to assign IDs from -1
+        // downwards, but skip any IDs that may already be in use.
+        $newTokenId = -1;
+        foreach ($compatTokens as $token) {
+            if (!\defined($token)) {
+                while (isset($usedTokenIds[$newTokenId])) {
+                    $newTokenId--;
+                }
+                \define($token, $newTokenId);
+                $newTokenId--;
+            }
+        }
+
+        $compatTokensDefined = true;
+    }
+
     /**
      * Creates the token map.
      *
@@ -344,20 +496,20 @@ class Lexer
         // 256 is the minimum possible token number, as everything below
         // it is an ASCII value
         for ($i = 256; $i < 1000; ++$i) {
-            if (T_DOUBLE_COLON === $i) {
+            if (\T_DOUBLE_COLON === $i) {
                 // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
                 $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
-            } elseif(T_OPEN_TAG_WITH_ECHO === $i) {
+            } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
                 // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
                 $tokenMap[$i] = Tokens::T_ECHO;
-            } elseif(T_CLOSE_TAG === $i) {
+            } elseif(\T_CLOSE_TAG === $i) {
                 // T_CLOSE_TAG is equivalent to ';'
                 $tokenMap[$i] = ord(';');
             } elseif ('UNKNOWN' !== $name = token_name($i)) {
                 if ('T_HASHBANG' === $name) {
                     // HHVM uses a special token for #! hashbang lines
                     $tokenMap[$i] = Tokens::T_INLINE_HTML;
-                } else if (defined($name = 'PhpParser\Parser\Tokens::' . $name)) {
+                } elseif (defined($name = Tokens::class . '::' . $name)) {
                     // Other tokens can be mapped directly
                     $tokenMap[$i] = constant($name);
                 }
@@ -366,13 +518,43 @@ class Lexer
 
         // HHVM uses a special token for numbers that overflow to double
         if (defined('T_ONUMBER')) {
-            $tokenMap[T_ONUMBER] = Tokens::T_DNUMBER;
+            $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
         }
         // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
         if (defined('T_COMPILER_HALT_OFFSET')) {
-            $tokenMap[T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
+            $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
         }
 
+        // Assign tokens for which we define compatibility constants, as token_name() does not know them.
+        $tokenMap[\T_FN] = Tokens::T_FN;
+        $tokenMap[\T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
+        $tokenMap[\T_NAME_QUALIFIED] = Tokens::T_NAME_QUALIFIED;
+        $tokenMap[\T_NAME_FULLY_QUALIFIED] = Tokens::T_NAME_FULLY_QUALIFIED;
+        $tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
+        $tokenMap[\T_MATCH] = Tokens::T_MATCH;
+        $tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR;
+        $tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE;
+        $tokenMap[\T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
+        $tokenMap[\T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG;
+        $tokenMap[\T_ENUM] = Tokens::T_ENUM;
+        $tokenMap[\T_READONLY] = Tokens::T_READONLY;
+
         return $tokenMap;
     }
+
+    private function createIdentifierTokenMap(): array {
+        // Based on semi_reserved production.
+        return array_fill_keys([
+            \T_STRING,
+            \T_STATIC, \T_ABSTRACT, \T_FINAL, \T_PRIVATE, \T_PROTECTED, \T_PUBLIC, \T_READONLY,
+            \T_INCLUDE, \T_INCLUDE_ONCE, \T_EVAL, \T_REQUIRE, \T_REQUIRE_ONCE, \T_LOGICAL_OR, \T_LOGICAL_XOR, \T_LOGICAL_AND,
+            \T_INSTANCEOF, \T_NEW, \T_CLONE, \T_EXIT, \T_IF, \T_ELSEIF, \T_ELSE, \T_ENDIF, \T_ECHO, \T_DO, \T_WHILE,
+            \T_ENDWHILE, \T_FOR, \T_ENDFOR, \T_FOREACH, \T_ENDFOREACH, \T_DECLARE, \T_ENDDECLARE, \T_AS, \T_TRY, \T_CATCH,
+            \T_FINALLY, \T_THROW, \T_USE, \T_INSTEADOF, \T_GLOBAL, \T_VAR, \T_UNSET, \T_ISSET, \T_EMPTY, \T_CONTINUE, \T_GOTO,
+            \T_FUNCTION, \T_CONST, \T_RETURN, \T_PRINT, \T_YIELD, \T_LIST, \T_SWITCH, \T_ENDSWITCH, \T_CASE, \T_DEFAULT,
+            \T_BREAK, \T_ARRAY, \T_CALLABLE, \T_EXTENDS, \T_IMPLEMENTS, \T_NAMESPACE, \T_TRAIT, \T_INTERFACE, \T_CLASS,
+            \T_CLASS_C, \T_TRAIT_C, \T_FUNC_C, \T_METHOD_C, \T_LINE, \T_FILE, \T_DIR, \T_NS_C, \T_HALT_COMPILER, \T_FN,
+            \T_MATCH,
+        ], true);
+    }
 }

lib/PhpParser/Lexer/Emulative.php

@@ -2,8 +2,250 @@
 
 namespace PhpParser\Lexer;
 
+use PhpParser\Error;
+use PhpParser\ErrorHandler;
+use PhpParser\Lexer;
+use PhpParser\Lexer\TokenEmulator\AttributeEmulator;
+use PhpParser\Lexer\TokenEmulator\EnumTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\ExplicitOctalEmulator;
+use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator;
+use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\MatchTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NullsafeTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
+use PhpParser\Lexer\TokenEmulator\ReadonlyFunctionTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\ReadonlyTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\ReverseEmulator;
+use PhpParser\Lexer\TokenEmulator\TokenEmulator;
 
-class Emulative extends \PhpParser\Lexer
+class Emulative extends Lexer
 {
-    /* No features requiring emulation have been added in PHP > 7.0 */
-}
+    const PHP_7_3 = '7.3dev';
+    const PHP_7_4 = '7.4dev';
+    const PHP_8_0 = '8.0dev';
+    const PHP_8_1 = '8.1dev';
+    const PHP_8_2 = '8.2dev';
+
+    /** @var mixed[] Patches used to reverse changes introduced in the code */
+    private $patches = [];
+
+    /** @var TokenEmulator[] */
+    private $emulators = [];
+
+    /** @var string */
+    private $targetPhpVersion;
+
+    /**
+     * @param mixed[] $options Lexer options. In addition to the usual options,
+     *                         accepts a 'phpVersion' string that specifies the
+     *                         version to emulate. Defaults to newest supported.
+     */
+    public function __construct(array $options = [])
+    {
+        $this->targetPhpVersion = $options['phpVersion'] ?? Emulative::PHP_8_2;
+        unset($options['phpVersion']);
+
+        parent::__construct($options);
+
+        $emulators = [
+            new FlexibleDocStringEmulator(),
+            new FnTokenEmulator(),
+            new MatchTokenEmulator(),
+            new CoaleseEqualTokenEmulator(),
+            new NumericLiteralSeparatorEmulator(),
+            new NullsafeTokenEmulator(),
+            new AttributeEmulator(),
+            new EnumTokenEmulator(),
+            new ReadonlyTokenEmulator(),
+            new ExplicitOctalEmulator(),
+            new ReadonlyFunctionTokenEmulator(),
+        ];
+
+        // Collect emulators that are relevant for the PHP version we're running
+        // and the PHP version we're targeting for emulation.
+        foreach ($emulators as $emulator) {
+            $emulatorPhpVersion = $emulator->getPhpVersion();
+            if ($this->isForwardEmulationNeeded($emulatorPhpVersion)) {
+                $this->emulators[] = $emulator;
+            } else if ($this->isReverseEmulationNeeded($emulatorPhpVersion)) {
+                $this->emulators[] = new ReverseEmulator($emulator);
+            }
+        }
+    }
+
+    public function startLexing(string $code, ?ErrorHandler $errorHandler = null) {
+        $emulators = array_filter($this->emulators, function($emulator) use($code) {
+            return $emulator->isEmulationNeeded($code);
+        });
+
+        if (empty($emulators)) {
+            // Nothing to emulate, yay
+            parent::startLexing($code, $errorHandler);
+            return;
+        }
+
+        $this->patches = [];
+        foreach ($emulators as $emulator) {
+            $code = $emulator->preprocessCode($code, $this->patches);
+        }
+
+        $collector = new ErrorHandler\Collecting();
+        parent::startLexing($code, $collector);
+        $this->sortPatches();
+        $this->fixupTokens();
+
+        $errors = $collector->getErrors();
+        if (!empty($errors)) {
+            $this->fixupErrors($errors);
+            foreach ($errors as $error) {
+                $errorHandler->handleError($error);
+            }
+        }
+
+        foreach ($emulators as $emulator) {
+            $this->tokens = $emulator->emulate($code, $this->tokens);
+        }
+    }
+
+    private function isForwardEmulationNeeded(string $emulatorPhpVersion): bool {
+        return version_compare(\PHP_VERSION, $emulatorPhpVersion, '<')
+            && version_compare($this->targetPhpVersion, $emulatorPhpVersion, '>=');
+    }
+
+    private function isReverseEmulationNeeded(string $emulatorPhpVersion): bool {
+        return version_compare(\PHP_VERSION, $emulatorPhpVersion, '>=')
+            && version_compare($this->targetPhpVersion, $emulatorPhpVersion, '<');
+    }
+
+    private function sortPatches()
+    {
+        // Patches may be contributed by different emulators.
+        // Make sure they are sorted by increasing patch position.
+        usort($this->patches, function($p1, $p2) {
+            return $p1[0] <=> $p2[0];
+        });
+    }
+
+    private function fixupTokens()
+    {
+        if (\count($this->patches) === 0) {
+            return;
+        }
+
+        // Load first patch
+        $patchIdx = 0;
+
+        list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+
+        // We use a manual loop over the tokens, because we modify the array on the fly
+        $pos = 0;
+        for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
+            $token = $this->tokens[$i];
+            if (\is_string($token)) {
+                if ($patchPos === $pos) {
+                    // Only support replacement for string tokens.
+                    assert($patchType === 'replace');
+                    $this->tokens[$i] = $patchText;
+
+                    // Fetch the next patch
+                    $patchIdx++;
+                    if ($patchIdx >= \count($this->patches)) {
+                        // No more patches, we're done
+                        return;
+                    }
+                    list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+                }
+
+                $pos += \strlen($token);
+                continue;
+            }
+
+            $len = \strlen($token[1]);
+            $posDelta = 0;
+            while ($patchPos >= $pos && $patchPos < $pos + $len) {
+                $patchTextLen = \strlen($patchText);
+                if ($patchType === 'remove') {
+                    if ($patchPos === $pos && $patchTextLen === $len) {
+                        // Remove token entirely
+                        array_splice($this->tokens, $i, 1, []);
+                        $i--;
+                        $c--;
+                    } else {
+                        // Remove from token string
+                        $this->tokens[$i][1] = substr_replace(
+                            $token[1], '', $patchPos - $pos + $posDelta, $patchTextLen
+                        );
+                        $posDelta -= $patchTextLen;
+                    }
+                } elseif ($patchType === 'add') {
+                    // Insert into the token string
+                    $this->tokens[$i][1] = substr_replace(
+                        $token[1], $patchText, $patchPos - $pos + $posDelta, 0
+                    );
+                    $posDelta += $patchTextLen;
+                } else if ($patchType === 'replace') {
+                    // Replace inside the token string
+                    $this->tokens[$i][1] = substr_replace(
+                        $token[1], $patchText, $patchPos - $pos + $posDelta, $patchTextLen
+                    );
+                } else {
+                    assert(false);
+                }
+
+                // Fetch the next patch
+                $patchIdx++;
+                if ($patchIdx >= \count($this->patches)) {
+                    // No more patches, we're done
+                    return;
+                }
+
+                list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+
+                // Multiple patches may apply to the same token. Reload the current one to check
+                // If the new patch applies
+                $token = $this->tokens[$i];
+            }
+
+            $pos += $len;
+        }
+
+        // A patch did not apply
+        assert(false);
+    }
+
+    /**
+     * Fixup line and position information in errors.
+     *
+     * @param Error[] $errors
+     */
+    private function fixupErrors(array $errors) {
+        foreach ($errors as $error) {
+            $attrs = $error->getAttributes();
+
+            $posDelta = 0;
+            $lineDelta = 0;
+            foreach ($this->patches as $patch) {
+                list($patchPos, $patchType, $patchText) = $patch;
+                if ($patchPos >= $attrs['startFilePos']) {
+                    // No longer relevant
+                    break;
+                }
+
+                if ($patchType === 'add') {
+                    $posDelta += strlen($patchText);
+                    $lineDelta += substr_count($patchText, "\n");
+                } else if ($patchType === 'remove') {
+                    $posDelta -= strlen($patchText);
+                    $lineDelta -= substr_count($patchText, "\n");
+                }
+            }
+
+            $attrs['startFilePos'] += $posDelta;
+            $attrs['endFilePos'] += $posDelta;
+            $attrs['startLine'] += $lineDelta;
+            $attrs['endLine'] += $lineDelta;
+            $error->setAttributes($attrs);
+        }
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php

@@ -0,0 +1,56 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class AttributeEmulator extends TokenEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_0;
+    }
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        return strpos($code, '#[') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way.
+        $line = 1;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            if ($tokens[$i] === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1] === '[') {
+                array_splice($tokens, $i, 2, [
+                    [\T_ATTRIBUTE, '#[', $line]
+                ]);
+                $c--;
+                continue;
+            }
+            if (\is_array($tokens[$i])) {
+                $line += substr_count($tokens[$i][1], "\n");
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        // TODO
+        return $tokens;
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        $pos = 0;
+        while (false !== $pos = strpos($code, '#[', $pos)) {
+            // Replace #[ with %[
+            $code[$pos] = '%';
+            $patches[] = [$pos, 'replace', '#'];
+            $pos += 2;
+        }
+        return $code;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php

@@ -0,0 +1,47 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class CoaleseEqualTokenEmulator extends TokenEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_7_4;
+    }
+
+    public function isEmulationNeeded(string $code): bool
+    {
+        return strpos($code, '??=') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $line = 1;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            if (isset($tokens[$i + 1])) {
+                if ($tokens[$i][0] === T_COALESCE && $tokens[$i + 1] === '=') {
+                    array_splice($tokens, $i, 2, [
+                        [\T_COALESCE_EQUAL, '??=', $line]
+                    ]);
+                    $c--;
+                    continue;
+                }
+            }
+            if (\is_array($tokens[$i])) {
+                $line += substr_count($tokens[$i][1], "\n");
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        // ??= was not valid code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/EnumTokenEmulator.php

@@ -0,0 +1,31 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class EnumTokenEmulator extends KeywordEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_1;
+    }
+
+    public function getKeywordString(): string
+    {
+        return 'enum';
+    }
+
+    public function getKeywordToken(): int
+    {
+        return \T_ENUM;
+    }
+
+    protected function isKeywordContext(array $tokens, int $pos): bool
+    {
+        return parent::isKeywordContext($tokens, $pos)
+            && isset($tokens[$pos + 2])
+            && $tokens[$pos + 1][0] === \T_WHITESPACE
+            && $tokens[$pos + 2][0] === \T_STRING;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/ExplicitOctalEmulator.php

@@ -0,0 +1,44 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+class ExplicitOctalEmulator extends TokenEmulator {
+    public function getPhpVersion(): string {
+        return Emulative::PHP_8_1;
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '0o') !== false || strpos($code, '0O') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            if ($tokens[$i][0] == \T_LNUMBER && $tokens[$i][1] === '0' &&
+                isset($tokens[$i + 1]) && $tokens[$i + 1][0] == \T_STRING &&
+                preg_match('/[oO][0-7]+(?:_[0-7]+)*/', $tokens[$i + 1][1])
+            ) {
+                $tokenKind = $this->resolveIntegerOrFloatToken($tokens[$i + 1][1]);
+                array_splice($tokens, $i, 2, [
+                    [$tokenKind, '0' . $tokens[$i + 1][1], $tokens[$i][2]],
+                ]);
+                $c--;
+            }
+        }
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int
+    {
+        $str = substr($str, 1);
+        $str = str_replace('_', '', $str);
+        $num = octdec($str);
+        return is_float($num) ? \T_DNUMBER : \T_LNUMBER;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // Explicit octals were not legal code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/FlexibleDocStringEmulator.php

@@ -0,0 +1,76 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class FlexibleDocStringEmulator extends TokenEmulator
+{
+    const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
+/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
+(?:.*\r?\n)*?
+(?<indentation>\h*)\2(?![a-zA-Z0-9_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
+REGEX;
+
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_7_3;
+    }
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        return strpos($code, '<<<') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // Handled by preprocessing + fixup.
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        // Not supported.
+        return $tokens;
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
+            // No heredoc/nowdoc found
+            return $code;
+        }
+
+        // Keep track of how much we need to adjust string offsets due to the modifications we
+        // already made
+        $posDelta = 0;
+        foreach ($matches as $match) {
+            $indentation = $match['indentation'][0];
+            $indentationStart = $match['indentation'][1];
+
+            $separator = $match['separator'][0];
+            $separatorStart = $match['separator'][1];
+
+            if ($indentation === '' && $separator !== '') {
+                // Ordinary heredoc/nowdoc
+                continue;
+            }
+
+            if ($indentation !== '') {
+                // Remove indentation
+                $indentationLen = strlen($indentation);
+                $code = substr_replace($code, '', $indentationStart + $posDelta, $indentationLen);
+                $patches[] = [$indentationStart + $posDelta, 'add', $indentation];
+                $posDelta -= $indentationLen;
+            }
+
+            if ($separator === '') {
+                // Insert newline as separator
+                $code = substr_replace($code, "\n", $separatorStart + $posDelta, 0);
+                $patches[] = [$separatorStart + $posDelta, 'remove', "\n"];
+                $posDelta += 1;
+            }
+        }
+
+        return $code;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php

@@ -0,0 +1,23 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class FnTokenEmulator extends KeywordEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_7_4;
+    }
+
+    public function getKeywordString(): string
+    {
+        return 'fn';
+    }
+
+    public function getKeywordToken(): int
+    {
+        return \T_FN;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/KeywordEmulator.php

@@ -0,0 +1,62 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+abstract class KeywordEmulator extends TokenEmulator
+{
+    abstract function getKeywordString(): string;
+    abstract function getKeywordToken(): int;
+
+    public function isEmulationNeeded(string $code): bool
+    {
+        return strpos(strtolower($code), $this->getKeywordString()) !== false;
+    }
+
+    protected function isKeywordContext(array $tokens, int $pos): bool
+    {
+        $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $pos);
+        return $previousNonSpaceToken === null || $previousNonSpaceToken[0] !== \T_OBJECT_OPERATOR;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        $keywordString = $this->getKeywordString();
+        foreach ($tokens as $i => $token) {
+            if ($token[0] === T_STRING && strtolower($token[1]) === $keywordString
+                    && $this->isKeywordContext($tokens, $i)) {
+                $tokens[$i][0] = $this->getKeywordToken();
+            }
+        }
+
+        return $tokens;
+    }
+
+    /**
+     * @param mixed[] $tokens
+     * @return array|string|null
+     */
+    private function getPreviousNonSpaceToken(array $tokens, int $start)
+    {
+        for ($i = $start - 1; $i >= 0; --$i) {
+            if ($tokens[$i][0] === T_WHITESPACE) {
+                continue;
+            }
+
+            return $tokens[$i];
+        }
+
+        return null;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        $keywordToken = $this->getKeywordToken();
+        foreach ($tokens as $i => $token) {
+            if ($token[0] === $keywordToken) {
+                $tokens[$i][0] = \T_STRING;
+            }
+        }
+
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/MatchTokenEmulator.php

@@ -0,0 +1,23 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class MatchTokenEmulator extends KeywordEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_0;
+    }
+
+    public function getKeywordString(): string
+    {
+        return 'match';
+    }
+
+    public function getKeywordToken(): int
+    {
+        return \T_MATCH;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/NullsafeTokenEmulator.php

@@ -0,0 +1,67 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class NullsafeTokenEmulator extends TokenEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_0;
+    }
+
+    public function isEmulationNeeded(string $code): bool
+    {
+        return strpos($code, '?->') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $line = 1;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            if ($tokens[$i] === '?' && isset($tokens[$i + 1]) && $tokens[$i + 1][0] === \T_OBJECT_OPERATOR) {
+                array_splice($tokens, $i, 2, [
+                    [\T_NULLSAFE_OBJECT_OPERATOR, '?->', $line]
+                ]);
+                $c--;
+                continue;
+            }
+
+            // Handle ?-> inside encapsed string.
+            if ($tokens[$i][0] === \T_ENCAPSED_AND_WHITESPACE && isset($tokens[$i - 1])
+                && $tokens[$i - 1][0] === \T_VARIABLE
+                && preg_match('/^\?->([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)/', $tokens[$i][1], $matches)
+            ) {
+                $replacement = [
+                    [\T_NULLSAFE_OBJECT_OPERATOR, '?->', $line],
+                    [\T_STRING, $matches[1], $line],
+                ];
+                if (\strlen($matches[0]) !== \strlen($tokens[$i][1])) {
+                    $replacement[] = [
+                        \T_ENCAPSED_AND_WHITESPACE,
+                        \substr($tokens[$i][1], \strlen($matches[0])),
+                        $line
+                    ];
+                }
+                array_splice($tokens, $i, 1, $replacement);
+                $c += \count($replacement) - 1;
+                continue;
+            }
+
+            if (\is_array($tokens[$i])) {
+                $line += substr_count($tokens[$i][1], "\n");
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        // ?-> was not valid code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -0,0 +1,105 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class NumericLiteralSeparatorEmulator extends TokenEmulator
+{
+    const BIN = '(?:0b[01]+(?:_[01]+)*)';
+    const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
+    const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
+    const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
+    const EXP = '(?:e[+-]?' . self::DEC . ')';
+    const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
+    const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
+
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_7_4;
+    }
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        return preg_match('~[0-9]_[0-9]~', $code)
+            || preg_match('~0x[0-9a-f]+_[0-9a-f]~i', $code);
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $codeOffset = 0;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            $tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
+
+            if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
+            assert($res, "No number at number token position");
+
+            $match = $matches[0];
+            $matchLen = \strlen($match);
+            if ($matchLen === $tokenLen) {
+                // Original token already holds the full number.
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $tokenKind = $this->resolveIntegerOrFloatToken($match);
+            $newTokens = [[$tokenKind, $match, $token[2]]];
+
+            $numTokens = 1;
+            $len = $tokenLen;
+            while ($matchLen > $len) {
+                $nextToken = $tokens[$i + $numTokens];
+                $nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
+                $nextTokenLen = \strlen($nextTokenText);
+
+                $numTokens++;
+                if ($matchLen < $len + $nextTokenLen) {
+                    // Split trailing characters into a partial token.
+                    assert(is_array($nextToken), "Partial token should be an array token");
+                    $partialText = substr($nextTokenText, $matchLen - $len);
+                    $newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
+                    break;
+                }
+
+                $len += $nextTokenLen;
+            }
+
+            array_splice($tokens, $i, $numTokens, $newTokens);
+            $c -= $numTokens - \count($newTokens);
+            $codeOffset += $matchLen;
+        }
+
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int
+    {
+        $str = str_replace('_', '', $str);
+
+        if (stripos($str, '0b') === 0) {
+            $num = bindec($str);
+        } elseif (stripos($str, '0x') === 0) {
+            $num = hexdec($str);
+        } elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
+            $num = octdec($str);
+        } else {
+            $num = +$str;
+        }
+
+        return is_float($num) ? T_DNUMBER : T_LNUMBER;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        // Numeric separators were not legal code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/ReadonlyFunctionTokenEmulator.php

@@ -0,0 +1,31 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+/*
+ * In PHP 8.1, "readonly(" was special cased in the lexer in order to support functions with
+ * name readonly. In PHP 8.2, this may conflict with readonly properties having a DNF type. For
+ * this reason, PHP 8.2 instead treats this as T_READONLY and then handles it specially in the
+ * parser. This emulator only exists to handle this special case, which is skipped by the
+ * PHP 8.1 ReadonlyTokenEmulator.
+ */
+class ReadonlyFunctionTokenEmulator extends KeywordEmulator {
+    public function getKeywordString(): string {
+        return 'readonly';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_READONLY;
+    }
+
+    public function getPhpVersion(): string {
+        return Emulative::PHP_8_2;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // Don't bother
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/ReadonlyTokenEmulator.php

@@ -0,0 +1,36 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class ReadonlyTokenEmulator extends KeywordEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_1;
+    }
+
+    public function getKeywordString(): string
+    {
+        return 'readonly';
+    }
+
+    public function getKeywordToken(): int
+    {
+        return \T_READONLY;
+    }
+
+    protected function isKeywordContext(array $tokens, int $pos): bool
+    {
+        if (!parent::isKeywordContext($tokens, $pos)) {
+            return false;
+        }
+        // Support "function readonly("
+        return !(isset($tokens[$pos + 1]) &&
+                 ($tokens[$pos + 1][0] === '(' ||
+                  ($tokens[$pos + 1][0] === \T_WHITESPACE &&
+                   isset($tokens[$pos + 2]) &&
+                   $tokens[$pos + 2][0] === '(')));
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/ReverseEmulator.php

@@ -0,0 +1,36 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+/**
+ * Reverses emulation direction of the inner emulator.
+ */
+final class ReverseEmulator extends TokenEmulator
+{
+    /** @var TokenEmulator Inner emulator */
+    private $emulator;
+
+    public function __construct(TokenEmulator $emulator) {
+        $this->emulator = $emulator;
+    }
+
+    public function getPhpVersion(): string {
+        return $this->emulator->getPhpVersion();
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return $this->emulator->isEmulationNeeded($code);
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        return $this->emulator->reverseEmulate($code, $tokens);
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        return $this->emulator->emulate($code, $tokens);
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        return $code;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/TokenEmulator.php

@@ -0,0 +1,25 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+/** @internal */
+abstract class TokenEmulator
+{
+    abstract public function getPhpVersion(): string;
+
+    abstract public function isEmulationNeeded(string $code): bool;
+
+    /**
+     * @return array Modified Tokens
+     */
+    abstract public function emulate(string $code, array $tokens): array;
+
+    /**
+     * @return array Modified Tokens
+     */
+    abstract public function reverseEmulate(string $code, array $tokens): array;
+
+    public function preprocessCode(string $code, array &$patches): string {
+        return $code;
+    }
+}

lib/PhpParser/NameContext.php

@@ -6,7 +6,8 @@ use PhpParser\Node\Name;
 use PhpParser\Node\Name\FullyQualified;
 use PhpParser\Node\Stmt;
 
-class NameContext {
+class NameContext
+{
     /** @var null|Name Current namespace */
     protected $namespace;
 
@@ -35,7 +36,7 @@ class NameContext {
      *
      * @param Name|null $namespace Null is the global namespace
      */
-    public function startNamespace(Name $namespace = null) {
+    public function startNamespace(?Name $namespace = null) {
         $this->namespace = $namespace;
         $this->origAliases = $this->aliases = [
             Stmt\Use_::TYPE_NORMAL   => [],
@@ -214,7 +215,7 @@ class NameContext {
 
         // Find shortest name
         $shortestName = null;
-        $shortestLength = INF;
+        $shortestLength = \INF;
         foreach ($possibleNames as $possibleName) {
             $length = strlen($possibleName->toCodeString());
             if ($length < $shortestLength) {
@@ -281,4 +282,4 @@ class NameContext {
         $shortName = substr($name, $nsSep + 1);
         return strtolower($ns) . '\\' . $shortName;
     }
-}
+}

lib/PhpParser/Node.php

@@ -90,13 +90,11 @@ interface Node
      *
      * @return Comment[]
      */
-    public function getComments(): array;
+    public function getComments() : array;
 
     /**
      * Gets the doc comment of the node.
      *
-     * The doc comment has to be the last comment associated with the node.
-     *
      * @return null|Comment\Doc Doc comment object or null
      */
     public function getDocComment();

lib/PhpParser/Node/Arg.php

@@ -2,10 +2,13 @@
 
 namespace PhpParser\Node;
 
+use PhpParser\Node\VariadicPlaceholder;
 use PhpParser\NodeAbstract;
 
 class Arg extends NodeAbstract
 {
+    /** @var Identifier|null Parameter name (for named parameters) */
+    public $name;
     /** @var Expr Value to pass */
     public $value;
     /** @var bool Whether to pass by ref */
@@ -20,15 +23,24 @@ class Arg extends NodeAbstract
      * @param bool  $byRef      Whether to pass by ref
      * @param bool  $unpack     Whether to unpack the argument
      * @param array $attributes Additional attributes
+     * @param Identifier|null $name Parameter name (for named parameters)
      */
-    public function __construct(Expr $value, bool $byRef = false, bool $unpack = false, array $attributes = []) {
-        parent::__construct($attributes);
+    public function __construct(
+        Expr $value, bool $byRef = false, bool $unpack = false, array $attributes = [],
+        ?Identifier $name = null
+    ) {
+        $this->attributes = $attributes;
+        $this->name = $name;
         $this->value = $value;
         $this->byRef = $byRef;
         $this->unpack = $unpack;
     }
 
     public function getSubNodeNames() : array {
-        return ['value', 'byRef', 'unpack'];
+        return ['name', 'value', 'byRef', 'unpack'];
+    }
+    
+    public function getType() : string {
+        return 'Arg';
     }
 }

lib/PhpParser/Node/Attribute.php

@@ -0,0 +1,34 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\Node;
+use PhpParser\NodeAbstract;
+
+class Attribute extends NodeAbstract
+{
+    /** @var Name Attribute name */
+    public $name;
+
+    /** @var Arg[] Attribute arguments */
+    public $args;
+
+    /**
+     * @param Node\Name $name       Attribute name
+     * @param Arg[]     $args       Attribute arguments
+     * @param array     $attributes Additional node attributes
+     */
+    public function __construct(Name $name, array $args = [], array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->name = $name;
+        $this->args = $args;
+    }
+
+    public function getSubNodeNames() : array {
+        return ['name', 'args'];
+    }
+
+    public function getType() : string {
+        return 'Attribute';
+    }
+}

lib/PhpParser/Node/AttributeGroup.php

@@ -0,0 +1,29 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\Node;
+use PhpParser\NodeAbstract;
+
+class AttributeGroup extends NodeAbstract
+{
+    /** @var Attribute[] Attributes */
+    public $attrs;
+
+    /**
+     * @param Attribute[] $attrs PHP attributes
+     * @param array $attributes Additional node attributes
+     */
+    public function __construct(array $attrs, array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->attrs = $attrs;
+    }
+
+    public function getSubNodeNames() : array {
+        return ['attrs'];
+    }
+
+    public function getType() : string {
+        return 'AttributeGroup';
+    }
+}

lib/PhpParser/Node/ComplexType.php

@@ -0,0 +1,14 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node;
+
+use PhpParser\NodeAbstract;
+
+/**
+ * This is a base class for complex types, including nullable types and union types.
+ *
+ * It does not provide any shared behavior and exists only for type-checking purposes.
+ */
+abstract class ComplexType extends NodeAbstract
+{
+}

lib/PhpParser/Node/Const_.php

@@ -4,9 +4,6 @@ namespace PhpParser\Node;
 
 use PhpParser\NodeAbstract;
 
-/**
- * @property Name $namespacedName Namespaced name (for class constants, if using NameResolver)
- */
 class Const_ extends NodeAbstract
 {
     /** @var Identifier Name */
@@ -14,6 +11,9 @@ class Const_ extends NodeAbstract
     /** @var Expr Value */
     public $value;
 
+    /** @var Name|null Namespaced name (if using NameResolver) */
+    public $namespacedName;
+
     /**
      * Constructs a const node for use in class const and const statements.
      *
@@ -22,7 +22,7 @@ class Const_ extends NodeAbstract
      * @param array             $attributes Additional attributes
      */
     public function __construct($name, Expr $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
         $this->value = $value;
     }
@@ -30,4 +30,8 @@ class Const_ extends NodeAbstract
     public function getSubNodeNames() : array {
         return ['name', 'value'];
     }
+
+    public function getType() : string {
+        return 'Const';
+    }
 }

lib/PhpParser/Node/Expr.php

@@ -6,4 +6,4 @@ use PhpParser\NodeAbstract;
 
 abstract class Expr extends NodeAbstract
 {
-}
+}

lib/PhpParser/Node/Expr/ArrayDimFetch.php

@@ -18,8 +18,8 @@ class ArrayDimFetch extends Expr
      * @param null|Expr $dim        Array index / dim
      * @param array     $attributes Additional attributes
      */
-    public function __construct(Expr $var, Expr $dim = null, array $attributes = []) {
-        parent::__construct($attributes);
+    public function __construct(Expr $var, ?Expr $dim = null, array $attributes = []) {
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->dim = $dim;
     }
@@ -27,4 +27,8 @@ class ArrayDimFetch extends Expr
     public function getSubNodeNames() : array {
         return ['var', 'dim'];
     }
+    
+    public function getType() : string {
+        return 'Expr_ArrayDimFetch';
+    }
 }

lib/PhpParser/Node/Expr/ArrayItem.php

@@ -12,6 +12,8 @@ class ArrayItem extends Expr
     public $value;
     /** @var bool Whether to assign by reference */
     public $byRef;
+    /** @var bool Whether to unpack the argument */
+    public $unpack;
 
     /**
      * Constructs an array item node.
@@ -21,14 +23,19 @@ class ArrayItem extends Expr
      * @param bool      $byRef      Whether to assign by reference
      * @param array     $attributes Additional attributes
      */
-    public function __construct(Expr $value, Expr $key = null, bool $byRef = false, array $attributes = []) {
-        parent::__construct($attributes);
+    public function __construct(Expr $value, ?Expr $key = null, bool $byRef = false, array $attributes = [], bool $unpack = false) {
+        $this->attributes = $attributes;
         $this->key = $key;
         $this->value = $value;
         $this->byRef = $byRef;
+        $this->unpack = $unpack;
     }
 
     public function getSubNodeNames() : array {
-        return ['key', 'value', 'byRef'];
+        return ['key', 'value', 'byRef', 'unpack'];
+    }
+
+    public function getType() : string {
+        return 'Expr_ArrayItem';
     }
 }

lib/PhpParser/Node/Expr/Array_.php

@@ -10,21 +10,25 @@ class Array_ extends Expr
     const KIND_LONG = 1;  // array() syntax
     const KIND_SHORT = 2; // [] syntax
 
-    /** @var ArrayItem[] Items */
+    /** @var (ArrayItem|null)[] Items */
     public $items;
 
     /**
      * Constructs an array node.
      *
-     * @param ArrayItem[] $items      Items of the array
+     * @param (ArrayItem|null)[] $items      Items of the array
      * @param array       $attributes Additional attributes
      */
     public function __construct(array $items = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->items = $items;
     }
 
     public function getSubNodeNames() : array {
         return ['items'];
     }
+    
+    public function getType() : string {
+        return 'Expr_Array';
+    }
 }

lib/PhpParser/Node/Expr/ArrowFunction.php

@@ -0,0 +1,79 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node\Expr;
+
+use PhpParser\Node;
+use PhpParser\Node\Expr;
+use PhpParser\Node\FunctionLike;
+
+class ArrowFunction extends Expr implements FunctionLike
+{
+    /** @var bool */
+    public $static;
+
+    /** @var bool */
+    public $byRef;
+
+    /** @var Node\Param[] */
+    public $params = [];
+
+    /** @var null|Node\Identifier|Node\Name|Node\ComplexType */
+    public $returnType;
+
+    /** @var Expr */
+    public $expr;
+    /** @var Node\AttributeGroup[] */
+    public $attrGroups;
+
+    /**
+     * @param array $subNodes   Array of the following optional subnodes:
+     *                          'static'     => false   : Whether the closure is static
+     *                          'byRef'      => false   : Whether to return by reference
+     *                          'params'     => array() : Parameters
+     *                          'returnType' => null    : Return type
+     *                          'expr'       => Expr    : Expression body
+     *                          'attrGroups' => array() : PHP attribute groups
+     * @param array $attributes Additional attributes
+     */
+    public function __construct(array $subNodes = [], array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->static = $subNodes['static'] ?? false;
+        $this->byRef = $subNodes['byRef'] ?? false;
+        $this->params = $subNodes['params'] ?? [];
+        $returnType = $subNodes['returnType'] ?? null;
+        $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
+        $this->expr = $subNodes['expr'];
+        $this->attrGroups = $subNodes['attrGroups'] ?? [];
+    }
+
+    public function getSubNodeNames() : array {
+        return ['attrGroups', 'static', 'byRef', 'params', 'returnType', 'expr'];
+    }
+
+    public function returnsByRef() : bool {
+        return $this->byRef;
+    }
+
+    public function getParams() : array {
+        return $this->params;
+    }
+
+    public function getReturnType() {
+        return $this->returnType;
+    }
+
+    public function getAttrGroups() : array {
+        return $this->attrGroups;
+    }
+
+    /**
+     * @return Node\Stmt\Return_[]
+     */
+    public function getStmts() : array {
+        return [new Node\Stmt\Return_($this->expr)];
+    }
+
+    public function getType() : string {
+        return 'Expr_ArrowFunction';
+    }
+}

lib/PhpParser/Node/Expr/Assign.php

@@ -19,7 +19,7 @@ class Assign extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }
@@ -27,4 +27,8 @@ class Assign extends Expr
     public function getSubNodeNames() : array {
         return ['var', 'expr'];
     }
+    
+    public function getType() : string {
+        return 'Expr_Assign';
+    }
 }

lib/PhpParser/Node/Expr/AssignOp.php

@@ -19,7 +19,7 @@ abstract class AssignOp extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }

lib/PhpParser/Node/Expr/AssignOp/BitwiseAnd.php

@@ -6,4 +6,7 @@ use PhpParser\Node\Expr\AssignOp;
 
 class BitwiseAnd extends AssignOp
 {
-}
+    public function getType() : string {
+        return 'Expr_AssignOp_BitwiseAnd';
+    }
+}

lib/PhpParser/Node/Expr/AssignOp/BitwiseOr.php

@@ -6,4 +6,7 @@ use PhpParser\Node\Expr\AssignOp;
 
 class BitwiseOr extends AssignOp
 {
-}
+    public function getType() : string {
+        return 'Expr_AssignOp_BitwiseOr';
+    }
+}

lib/PhpParser/Node/Expr/AssignOp/BitwiseXor.php

@@ -6,4 +6,7 @@ use PhpParser\Node\Expr\AssignOp;
 
 class BitwiseXor extends AssignOp
 {
-}
+    public function getType() : string {
+        return 'Expr_AssignOp_BitwiseXor';
+    }
+}