Update lexer implementations and tests

Each token is now represented by a Token object.

.editorconfig

@@ -1,9 +0,0 @@
-root = true
-
-[*.y]
-charset = utf-8
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
-indent_style = space
-indent_size = 4

.gitattributes

@@ -1,10 +0,0 @@
-/.github         export-ignore
-/doc             export-ignore
-/test            export-ignore
-/test_old        export-ignore
-.editorconfig    export-ignore
-.gitattributes   export-ignore
-.gitignore       export-ignore
-CHANGELOG.md     export-ignore
-phpunit.xml.dist export-ignore
-UPGRADE-*.md     export-ignore

.github/workflows/main.yml

@@ -1,90 +0,0 @@
-# https://help.github.com/en/categories/automating-your-workflow-with-github-actions
-name: Main
-on:
-  push:
-  pull_request:
-
-jobs:
-  tests_70:
-    runs-on: "ubuntu-latest"
-    name: "PHP 7.0 Unit Tests"
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          coverage: "xdebug"
-          php-version: "7.0"
-          tools: composer:v2
-      - name: "Install dependencies"
-        run: |
-          composer require php-coveralls/php-coveralls:^2.2 --dev --no-update
-          composer update --no-progress --prefer-dist
-      - name: "Tests"
-        run: "php vendor/bin/phpunit --coverage-clover build/logs/clover.xml"
-      - name: Coveralls
-        env:
-          COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: "php vendor/bin/php-coveralls"
-        if: ${{ success() }}
-  tests:
-    runs-on: "ubuntu-latest"
-    name: "PHP ${{ matrix.php-version }} Unit Tests"
-    strategy:
-      matrix:
-        php-version:
-          - "7.1"
-          - "7.2"
-          - "7.3"
-          - "7.4"
-          - "8.0"
-          - "8.1"
-        include:
-          - php-version: "8.1"
-            flags: "--ignore-platform-req=php"
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          coverage: "none"
-          php-version: "${{ matrix.php-version }}"
-          tools: composer:v2
-      - name: "Install dependencies"
-        run: "composer update --no-progress --prefer-dist ${{ matrix.flags }}"
-      - name: "PHPUnit"
-        run: "php vendor/bin/phpunit"
-  test_old_73_80:
-    runs-on: "ubuntu-latest"
-    name: "PHP 7.3 Code on PHP 8.0 Integration Tests"
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          coverage: "none"
-          php-version: "8.0"
-          tools: composer:v2
-      - name: "Install PHP 8 dependencies"
-        run: "composer update --no-progress --prefer-dist"
-      - name: "Tests"
-        run: "test_old/run-php-src.sh 7.3.21"
-  test_old_80_70:
-    runs-on: "ubuntu-latest"
-    name: "PHP 8.0 Code on PHP 7.0 Integration Tests"
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v2"
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          coverage: "none"
-          php-version: "7.0"
-          tools: composer:v2
-      - name: "Install PHP 8 dependencies"
-        run: "composer update --no-progress --prefer-dist"
-      - name: "Tests"
-        run: "test_old/run-php-src.sh 8.0.0"

.gitignore

@@ -2,4 +2,3 @@ vendor/
 composer.lock
 grammar/kmyacc.exe
 grammar/y.output
-.phpunit.result.cache

.travis.yml

@@ -0,0 +1,36 @@
+language: php
+dist: xenial
+sudo: false
+
+cache:
+  directories:
+    - $HOME/.composer/cache
+
+php:
+  - 7.0
+  - 7.1
+  - 7.2
+  - 7.3
+  - 7.4snapshot
+  - nightly
+
+install:
+  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then composer require satooshi/php-coveralls '~1.0'; fi
+  - |
+    if [ $TRAVIS_PHP_VERSION = 'nightly' ]; then
+      composer install --prefer-dist --ignore-platform-reqs;
+    else
+      composer install --prefer-dist;
+    fi
+
+matrix:
+  allow_failures:
+    - php: nightly
+  fast_finish: true
+
+script:
+  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then vendor/bin/phpunit --coverage-clover build/logs/clover.xml; else vendor/bin/phpunit; fi
+  - if [ $TRAVIS_PHP_VERSION = '7.2' ]; then test_old/run-php-src.sh; fi
+
+after_success:
+  - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then php vendor/bin/coveralls; fi

CHANGELOG.md

@@ -1,302 +1,8 @@
-Version 4.13.3-dev
-------------------
+Version 4.2.3-dev
+-----------------
 
 Nothing yet.
 
-Version 4.13.2 (2021-11-30)
----------------------------
-
-### Added
-
-* Added builders for enums and enum cases.
-
-### Fixed
-
-* NullsafeMethodCall now extends from CallLike.
-* The `namespacedName` property populated by the `NameResolver` is now declared on relevant nodes,
-  to avoid a dynamic property deprecation warning with PHP 8.2.
-
-Version 4.13.1 (2021-11-03)
----------------------------
-
-### Fixed
-
-* Support reserved keywords as enum cases.
-* Support array unpacking in constant expression evaluator.
-
-Version 4.13.0 (2021-09-20)
----------------------------
-
-### Added
-
-* [PHP 8.1] Added support for intersection types using a new `IntersectionType` node. Additionally
-  a `ComplexType` parent class for `NullableType`, `UnionType` and `IntersectionType` has been
-  added.
-* [PHP 8.1] Added support for explicit octal literals.
-* [PHP 8.1] Added support for first-class callables. These are represented using a call whose first
-  argument is a `VariadicPlaceholder`. The representation is intended to be forward-compatible with
-  partial function application, just like the PHP feature itself. Call nodes now extend from
-  `Expr\CallLike`, which provides an `isFirstClassCallable()` method to determine whether a
-  placeholder id present. `getArgs()` can be used to assert that the call is not a first-class
-  callable and returns `Arg[]` rather than `array<Arg|VariadicPlaceholder>`.
-
-### Fixed
-
-* Multiple modifiers for promoted properties are now accepted. In particular this allows something
-  like `public readonly` for promoted properties.
-* Formatting-preserving pretty printing for comments in array literals has been fixed.
-
-Version 4.12.0 (2021-07-21)
----------------------------
-
-### Added
-
-* [PHP 8.1] Added support for readonly properties (through a new `MODIFIER_READONLY`).
-* [PHP 8.1] Added support for final class constants.
-
-### Fixed
-
-* Fixed compatibility with PHP 8.1. `&` tokens are now canonicalized to the
-  `T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG` and `T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG` tokens
-  used in PHP 8.1. This happens unconditionally, regardless of whether the emulative lexer is used.
-
-Version 4.11.0 (2021-07-03)
----------------------------
-
-### Added
-
-* `BuilderFactory::args()` now accepts named arguments.
-* `BuilderFactory::attribute()` has been added.
-* An `addAttribute()` method accepting an `Attribute` or `AttributeGroup` has been adde to all
-  builders that accept attributes, such as `Builder\Class_`.
-
-### Fixed
-
-* `NameResolver` now handles enums.
-* `PrettyPrinter` now prints backing enum type.
-* Builder methods for types now property handle `never` type.
-
-Version 4.10.5 (2021-05-03)
----------------------------
-
-### Added
-
-* [PHP 8.1] Added support for enums. These are represented using the `Stmt\Enum_` and
-  `Stmt\EnumCase` nodes.
-* [PHP 8.1] Added support for never type. This type will now be returned as an `Identifier` rather
-  than `Name`.
-* Added `ClassConst` builder.
-
-### Changed
-
-* Non-UTF-8 code units in strings will now be hex-encoded.
-
-### Fixed
-
-* Fixed precedence of arrow functions.
-
-Version 4.10.4 (2020-12-20)
----------------------------
-
-### Fixed
-
-* Fixed position information for variable-variables (#741).
-* Fixed position information for traits/interfaces preceded by if statement (#738).
-
-Version 4.10.3 (2020-12-03)
----------------------------
-
-### Fixed
-
-* Fixed formatting-preserving pretty printing for `"{$x}"`.
-* Ternary expressions are now treated as non-associative in the pretty printer, in order to
-  generate code that is compatible with the parentheses requirement introduced in PHP 8.
-* Removed no longer necessary `error_clear_last()` call in lexer, which may interfere with fatal
-  error handlers if invoked during shutdown.
-
-
-Version 4.10.2 (2020-09-26)
-------------------
-
-### Fixed
-
-* Fixed check for token emulation conflicts with other libraries.
-
-Version 4.10.1 (2020-09-23)
----------------------------
-
-### Added
-
-* Added support for recovering from a missing semicolon after a property or class constant
-  declaration.
-
-### Fixed
-
-* Fix spurious whitespace in formatting-preserving pretty printer when both removing and adding
-  elements at the start of a list.
-* Fix incorrect case-sensitivity in keyword token emulation.
-
-Version 4.10.0 (2020-09-19)
----------------------------
-
-### Added
-
-* [PHP 8.0] Added support for attributes. These are represented using a new `AttributeGroup` node
-  containing `Attribute` nodes. A new `attrGroups` subnode is available on all node types that
-  support attributes, i.e. `Stmt\Class_`, `Stmt\Trait_`, `Stmt\Interface_`, `Stmt\Function_`,
-  `Stmt\ClassMethod`, `Stmt\ClassConst`, `Stmt\Property`, `Expr\Closure`, `Expr\ArrowFunction` and
-  `Param`.
-* [PHP 8.0] Added support for nullsafe properties inside interpolated strings, in line with an
-  upstream change.
-
-### Fixed
-
-* Improved compatibility with other libraries that use forward compatibility defines for PHP tokens.
-
-Version 4.9.1 (2020-08-30)
---------------------------
-
-### Added
-
-* Added support for removing the first element of a list to the formatting-preserving pretty
-  printer.
-
-### Fixed
-
-* Allow member modifiers as part of namespaced names. These were missed when support for other
-  keywords was added.
-
-Version 4.9.0 (2020-08-18)
---------------------------
-
-### Added
-
-* [PHP 8.0] Added support for named arguments, represented using a new `name` subnode on `Arg`.
-* [PHP 8.0] Added support for static return type, represented like a normal class return type.
-* [PHP 8.0] Added support for throw expression, represented using a new `Expr\Throw_` node. For
-  backwards compatibility reasons, throw expressions in statement context continue to be
-  represented using `Stmt\Throw_`.
-* [PHP 8.0] Added support for keywords as parts of namespaced names.
-
-### Fixed
-
-* Emit parentheses for class constant fetch with complex left-hand-side.
-* Emit parentheses for new/instanceof on complex class expression.
-
-Version 4.8.0 (2020-08-09)
---------------------------
-
-### Added
-
-* [PHP 8.0] Added support for nullsafe operator, represented using the new
-  `Expr\NullsafePropertyFetch` and `Expr\NullsafeMethodCall` nodes.
-* Added `phpVersion` option to the emulative lexer, which allows controlling the target version to
-  emulate (defaults to the latest available, currently PHP 8.0). This is useful to parse code that
-  uses reserved keywords from newer PHP versions as identifiers.
-
-Version 4.7.0 (2020-07-25)
---------------------------
-
-### Added
-
-* Add `ParentConnectingVisitor` and `NodeConnectingVisitor` classes.
-* [PHP 8.0] Added support for match expressions. These are represented using a new `Expr\Match_`
-  containing `MatchArm`s.
-* [PHP 8.0] Added support for trailing comma in closure use lists.
-
-### Fixed
-
-* Fixed missing error for unterminated comment with trailing newline (#688).
-* Compatibility with PHP 8.0 has been restored: Namespaced names are now always represented by
-  `T_NAME_*` tokens, using emulationg on older PHP versions. Full support for reserved keywords
-  in namespaced names is not yet present.
-
-Version 4.6.0 (2020-07-02)
---------------------------
-
-### Added
-
-* [PHP 8.0] Added support for trailing commas in parameter lists.
-* [PHP 8.0] Added support for constructor promotion. The parameter visibility is stored in
-  `Node\Param::$flags`.
-
-### Fixed
-
-* Comment tokens now always follow the PHP 8 interpretation, and do not include trailing
-  whitespace.
-* As a result of the previous change, some whitespace issues when inserting a statement into a
-  method containing only a comment, and using the formatting-preserving pretty printer, have been
-  resolved.
-
-Version 4.5.0 (2020-06-03)
---------------------------
-
-### Added
-
-* [PHP 8.0] Added support for the mixed type. This means `mixed` types are now parsed as an
-  `Identifier` rather than a `Name`.
-* [PHP 8.0] Added support for catching without capturing the exception. This means that
-  `Catch_::$var` may now be null.
-
-Version 4.4.0 (2020-04-10)
---------------------------
-
-### Added
-
-* Added support for passing union types in builders.
-* Added end line, token position and file position information for comments.
-* Added `getProperty()` method to `ClassLike` nodes.
-
-### Fixed
-
-* Fixed generation of invalid code when using the formatting preserving pretty printer, and
-  inserting code next to certain nop statements. The formatting is still ugly though.
-* `getDocComment()` no longer requires that the very last comment before a node be a doc comment.
-  There may not be non-doc comments between the doc comment and the declaration.
-* Allowed arbitrary expressions in `isset()` and `list()`, rather than just variables.
-  In particular, this allows `isset(($x))`, which is legal PHP code.
-* [PHP 8.0] Add support for [variable syntax tweaks RFC](https://wiki.php.net/rfc/variable_syntax_tweaks).
-
-Version 4.3.0 (2019-11-08)
---------------------------
-
-### Added
-
-* [PHP 8.0] Added support for union types using a new `UnionType` node.
-
-Version 4.2.5 (2019-10-25)
---------------------------
-
-### Changed
-
-* Tests and documentation are no longer included in source archives. They can still be accessed
-  by cloning the repository.
-* php-yacc is now used to generate the parser. This has no impact on users of the library.
-
-Version 4.2.4 (2019-09-01)
---------------------------
-
-### Added
-
-* Added getProperties(), getConstants() and getTraitUses() to ClassLike. (#629, #630)
-
-### Fixed
-
-* Fixed flexible heredoc emulation to check for digits after the end label. This synchronizes
-  behavior with the upcoming PHP 7.3.10 release.
-
-Version 4.2.3 (2019-08-12)
---------------------------
-
-### Added
-
-* [PHP 7.4] Add support for numeric literal separators. (#615)
-
-### Fixed
-
-* Fixed resolution of return types for arrow functions. (#613)
-* Fixed compatibility with PHP 7.4.
-
 Version 4.2.2 (2019-05-25)
 --------------------------
 

LICENSE

@@ -1,29 +1,31 @@
-BSD 3-Clause License
+Copyright (c) 2011-2018 by Nikita Popov.
 
-Copyright (c) 2011, Nikita Popov
-All rights reserved.
+Some rights reserved.
 
 Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+modification, are permitted provided that the following conditions are
+met:
 
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
 
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
 
-3. Neither the name of the copyright holder nor the names of its
-   contributors may be used to endorse or promote products derived from
-   this software without specific prior written permission.
+    * The names of the contributors may not be used to endorse or
+      promote products derived from this software without specific
+      prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1,12 +1,12 @@
 PHP Parser
 ==========
 
-[![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
+[![Build Status](https://travis-ci.org/nikic/PHP-Parser.svg?branch=master)](https://travis-ci.org/nikic/PHP-Parser) [![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
 
-This is a PHP 5.2 to PHP 8.0 parser written in PHP. Its purpose is to simplify static code analysis and
+This is a PHP 5.2 to PHP 7.3 parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
-[**Documentation for version 4.x**][doc_master] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 8.0).
+[**Documentation for version 4.x**][doc_master] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 7.3).
 
 [Documentation for version 3.x][doc_3_x] (unsupported; for running on PHP >= 5.5; for parsing PHP 5.2 to PHP 7.2).
 
@@ -15,7 +15,7 @@ Features
 
 The main features provided by this library are:
 
- * Parsing PHP 5, PHP 7, and PHP 8 code into an abstract syntax tree (AST).
+ * Parsing PHP 5 and PHP 7 code into an abstract syntax tree (AST).
    * Invalid code can be parsed into a partial AST.
    * The AST contains accurate location information.
  * Dumping the AST in human-readable form.
@@ -215,7 +215,7 @@ Component documentation:
  * [JSON representation](doc/component/JSON_representation.markdown)
    * JSON encoding and decoding of ASTs
  * [Performance](doc/component/Performance.markdown)
-   * Disabling Xdebug
+   * Disabling XDebug
    * Reusing objects
    * Garbage collection impact
  * [Frequently asked questions](doc/component/FAQ.markdown)

bin/php-parse

@@ -10,7 +10,7 @@ foreach ([__DIR__ . '/../../../autoload.php', __DIR__ . '/../vendor/autoload.php
 
 ini_set('xdebug.max_nesting_level', 3000);
 
-// Disable Xdebug var_dump() output truncation
+// Disable XDebug var_dump() output truncation
 ini_set('xdebug.var_display_max_children', -1);
 ini_set('xdebug.var_display_max_data', -1);
 ini_set('xdebug.var_display_max_depth', -1);

composer.json

@@ -14,15 +14,15 @@
     ],
     "require": {
         "php": ">=7.0",
+        "ext-json": "*",
         "ext-tokenizer": "*"
     },
     "require-dev": {
-        "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0",
-        "ircmaxell/php-yacc": "^0.0.7"
+        "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0"
     },
     "extra": {
         "branch-alias": {
-            "dev-master": "4.9-dev"
+            "dev-master": "4.2-dev"
         }
     },
     "autoload": {

doc/0_Introduction.markdown

@@ -1,7 +1,7 @@
 Introduction
 ============
 
-This project is a PHP 5.2 to PHP 8.0 parser **written in PHP itself**.
+This project is a PHP 5.2 to PHP 7.3 parser **written in PHP itself**.
 
 What is this for?
 -----------------
@@ -26,15 +26,11 @@ programmatic PHP code analysis are incidentally PHP developers, not C developers
 What can it parse?
 ------------------
 
-The parser supports parsing PHP 5.2-8.0, with the following exceptions:
-
- * Namespaced names containing whitespace (e.g. `Foo \ Bar` instead of `Foo\Bar`) are not supported.
-   These are illegal in PHP 8, but are legal in earlier version. However, PHP-Parser does not
-   support them for any version.
+The parser supports parsing PHP 5.2-7.3.
 
 As the parser is based on the tokens returned by `token_get_all` (which is only able to lex the PHP
 version it runs on), additionally a wrapper for emulating tokens from newer versions is provided.
-This allows to parse PHP 7.4 source code running on PHP 7.0, for example. This emulation is somewhat
+This allows to parse PHP 7.3 source code running on PHP 7.0, for example. This emulation is somewhat
 hacky and not perfect, but it should work well on any sane code.
 
 What output does it produce?

doc/2_Usage_of_basic_components.markdown

@@ -19,7 +19,7 @@ ini_set('xdebug.max_nesting_level', 3000);
 ```
 
 This ensures that there will be no errors when traversing highly nested node trees. However, it is
-preferable to disable Xdebug completely, as it can easily make this library more than five times
+preferable to disable XDebug completely, as it can easily make this library more than five times
 slower.
 
 Parsing
@@ -77,7 +77,7 @@ A parser instance can be reused to parse multiple files.
 Node dumping
 ------------
 
-To dump the abstract syntax tree in human readable form, a `NodeDumper` can be used:
+To dump the abstact syntax tree in human readable form, a `NodeDumper` can be used:
 
 ```php
 <?php

doc/README.md

@@ -39,7 +39,7 @@ Component documentation
   * [JSON representation](component/JSON_representation.markdown)
     * JSON encoding and decoding of ASTs
   * [Performance](component/Performance.markdown)
-    * Disabling Xdebug
+    * Disabling XDebug
     * Reusing objects
     * Garbage collection impact
   * [Frequently asked questions](component/FAQ.markdown)

doc/component/Error_handling.markdown

@@ -36,7 +36,7 @@ if ($e->hasColumnInfo()) {
     echo $e->getRawMessage() . ' from ' . $e->getStartLine() . ':' . $e->getStartColumn($code)
         . ' to ' . $e->getEndLine() . ':' . $e->getEndColumn($code);
     // or:
-    echo $e->getMessageWithColumnInfo($code);
+    echo $e->getMessageWithColumnInfo();
 } else {
     echo $e->getMessage();
 }

doc/component/FAQ.markdown

@@ -7,21 +7,28 @@ Frequently Asked Questions
 How can the parent of a node be obtained?
 -----
 
-The AST does not store parent nodes by default. However, the `ParentConnectingVisitor` can be used to achieve this:
+The AST does not store parent nodes by default. However, it is easy to add a custom parent node
+attribute using a custom node visitor:
 
 ```php
-use PhpParser\NodeTraverser;
-use PhpParser\NodeVisitor\ParentConnectingVisitor;
-use PhpParser\ParserFactory;
+use PhpParser\Node;
+use PhpParser\NodeVisitorAbstract;
 
-$code = '...';
-
-$traverser = new NodeTraverser;
-$traverser->addVisitor(new ParentConnectingVisitor);
-
-$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
-$ast    = $parser->parse($code);
-$ast    = $traverser->traverse($ast);
+class ParentConnector extends NodeVisitorAbstract {
+    private $stack;
+    public function beforeTraverse(array $nodes) {
+        $this->stack = [];
+    }
+    public function enterNode(Node $node) {
+        if (!empty($this->stack)) {
+            $node->setAttribute('parent', $this->stack[count($this->stack)-1]);
+        }
+        $this->stack[] = $node;
+    }
+    public function leaveNode(Node $node) {
+        array_pop($this->stack);
+    }
+}
 ```
 
 After running this visitor, the parent node can be obtained through `$node->getAttribute('parent')`.
@@ -29,27 +36,33 @@ After running this visitor, the parent node can be obtained through `$node->getA
 How can the next/previous sibling of a node be obtained?
 -----
 
-Again, siblings are not stored by default, but the `NodeConnectingVisitor` can be used to store
-the previous / next node with a common parent as well:
+Again, siblings are not stored by default, but the visitor from the previous entry can be easily
+extended to store the previous / next node with a common parent as well:
 
 ```php
-use PhpParser\NodeTraverser;
-use PhpParser\NodeVisitor\NodeConnectingVisitor;
-use PhpParser\ParserFactory;
+use PhpParser\Node;
+use PhpParser\NodeVisitorAbstract;
 
-$code = '...';
-
-$traverser = new NodeTraverser;
-$traverser->addVisitor(new NodeConnectingVisitor);
-
-$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
-$ast    = $parser->parse($code);
-$ast    = $traverser->traverse($ast);
+class NodeConnector extends NodeVisitorAbstract {
+    private $stack;
+    private $prev;
+    public function beforeTraverse(array $nodes) {
+        $this->stack = [];
+        $this->prev = null;
+    }
+    public function enterNode(Node $node) {
+        if (!empty($this->stack)) {
+            $node->setAttribute('parent', $this->stack[count($this->stack)-1]);
+        }
+        if ($this->prev && $this->prev->getAttribute('parent') == $node->getAttribute('parent')) {
+            $node->setAttribute('prev', $this->prev);
+            $this->prev->setAttribute('next', $node);
+        }
+        $this->stack[] = $node;
+    }
+    public function leaveNode(Node $node) {
+        $this->prev = $node;
+        array_pop($this->stack);
+    }
+}
 ```
-
-After running this visitor, the parent node can be obtained through `$node->getAttribute('parent')`,
-the previous node can be obtained through `$node->getAttribute('previous')`, and the next node can be
-obtained through `$node->getAttribute('next')`.
-
-`ParentConnectingVisitor` and `NodeConnectingVisitor` should not be used at the same time. The latter
-includes the functionality of the former.

doc/component/Performance.markdown

@@ -8,20 +8,20 @@ described in the following.
 Xdebug
 ------
 
-Running PHP with Xdebug adds a lot of overhead, especially for code that performs many method calls.
-Just by loading Xdebug (without enabling profiling or other more intrusive Xdebug features), you
+Running PHP with XDebug adds a lot of overhead, especially for code that performs many method calls.
+Just by loading XDebug (without enabling profiling or other more intrusive XDebug features), you
 can expect that code using PHP-Parser will be approximately *five times slower*.
 
-As such, you should make sure that Xdebug is not loaded when using this library. Note that setting
-the `xdebug.default_enable=0` ini option does *not* disable Xdebug. The *only* way to disable
-Xdebug is to not load the extension in the first place.
+As such, you should make sure that XDebug is not loaded when using this library. Note that setting
+the `xdebug.default_enable=0` ini option does *not* disable XDebug. The *only* way to disable
+XDebug is to not load the extension in the first place.
 
-If you are building a command-line utility for use by developers (who often have Xdebug enabled),
-you may want to consider automatically restarting PHP with Xdebug unloaded. The
+If you are building a command-line utility for use by developers (who often have XDebug enabled),
+you may want to consider automatically restarting PHP with XDebug unloaded. The
 [composer/xdebug-handler](https://github.com/composer/xdebug-handler) package can be used to do
 this.
 
-If you do run with Xdebug, you may need to increase the `xdebug.max_nesting_level` option to a
+If you do run with XDebug, you may need to increase the `xdebug.max_nesting_level` option to a
 higher level, such as 3000. While the parser itself is recursion free, most other code working on
 the AST uses recursion and will generate an error if the value of this option is too low.
 

doc/component/Walking_the_AST.markdown

@@ -193,7 +193,7 @@ anonymous classes), you know that once you've seen a class declaration, there is
 checking all it's child nodes, because PHP does not allow nesting classes. In this case, you can
 instruct the traverser to not recurse into the class node:
 
-```php
+```
 private $classes = [];
 public function enterNode(Node $node) {
     if ($node instanceof Node\Stmt\Class_) {
@@ -211,7 +211,7 @@ after finding it. For example, if you are looking for the node of a class with a
 discounting exotic cases like conditionally defining a class two times), you can stop traversal
 once you found it:
 
-```php
+```
 private $class = null;
 public function enterNode(Node $node) {
     if ($node instanceof Node\Stmt\Class_ &&
@@ -251,7 +251,7 @@ Stmt_Return(
 
 the following method calls will be performed:
 
-```php
+```
 $visitorA->enterNode(Stmt_Return)
 $visitorB->enterNode(Stmt_Return)
 $visitorA->enterNode(Expr_Variable)
@@ -295,7 +295,7 @@ Simple node finding
 
 While the node visitor mechanism is very flexible, creating a node visitor can be overly cumbersome
 for minor tasks. For this reason a `NodeFinder` is provided, which can find AST nodes that either
-satisfy a certain callback, or which are instances of a certain node type. A couple of examples are
+satisfy a certain callback, or which are instanced of a certain node type. A couple of examples are
 shown in the following:
 
 ```php
@@ -312,7 +312,7 @@ $extendingClasses = $nodeFinder->find($stmts, function(Node $node) {
         && $node->extends !== null;
 });
 
-// Find first class occurring in the AST. Returns null if no class exists.
+// Find first class occuring in the AST. Returns null if no class exists.
 $class = $nodeFinder->findFirstInstanceOf($stmts, Node\Stmt\Class_::class);
 
 // Find first class that has name $name

grammar/README.md

@@ -21,10 +21,8 @@ applied to it:
 Building the parser
 ===================
 
-Run `php grammar/rebuildParsers.php` to rebuild the parsers. Additional options:
+In order to rebuild the parser, you need [moriyoshi's fork of kmyacc](https://github.com/moriyoshi/kmyacc-forked).
+After you compiled/installed it, run the `rebuildParsers.php` script.
 
- * The `KMYACC` environment variable can be used to specify an alternative `kmyacc` binary.
-   By default the `phpyacc` dev dependency will be used. To use the original `kmyacc`, you
-   need to compile [moriyoshi's fork](https://github.com/moriyoshi/kmyacc-forked).
- * The `--debug` option enables emission of debug symbols and creates the `y.output` file.
- * The `--keep-tmp-grammar` option preserves the preprocessed grammar file.
+By default only the `Parser.php` is built. If you want to additionally emit debug symbols and create `y.output`, run the
+script with `--debug`. If you want to retain the preprocessed grammar pass `--keep-tmp-grammar`.

grammar/php5.y

@@ -20,11 +20,6 @@ top_statement_list:
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
-ampersand:
-      T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
-    | T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG
-;
-
 reserved_non_modifiers:
       T_INCLUDE | T_INCLUDE_ONCE | T_EVAL | T_REQUIRE | T_REQUIRE_ONCE | T_LOGICAL_OR | T_LOGICAL_XOR | T_LOGICAL_AND
     | T_INSTANCEOF | T_NEW | T_CLONE | T_EXIT | T_IF | T_ELSEIF | T_ELSE | T_ENDIF | T_ECHO | T_DO | T_WHILE
@@ -33,7 +28,6 @@ reserved_non_modifiers:
     | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT
     | T_BREAK | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE | T_CLASS
     | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER | T_FN
-    | T_MATCH
 ;
 
 semi_reserved:
@@ -54,14 +48,13 @@ reserved_non_modifiers_identifier:
       reserved_non_modifiers                                { $$ = Node\Identifier[$1]; }
 ;
 
-namespace_name:
-      T_STRING                                              { $$ = Name[$1]; }
-    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
+namespace_name_parts:
+      T_STRING                                              { init($1); }
+    | namespace_name_parts T_NS_SEPARATOR T_STRING          { push($1, $3); }
 ;
 
-legacy_namespace_name:
-      namespace_name                                        { $$ = $1; }
-    | T_NAME_FULLY_QUALIFIED                                { $$ = Name[substr($1, 1)]; }
+namespace_name:
+      namespace_name_parts                                  { $$ = Name[$1]; }
 ;
 
 plain_variable:
@@ -97,11 +90,16 @@ use_type:
     | T_CONST                                               { $$ = Stmt\Use_::TYPE_CONSTANT; }
 ;
 
+/* Using namespace_name_parts here to avoid s/r conflict on T_NS_SEPARATOR */
 group_use_declaration:
-      T_USE use_type legacy_namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
-          { $$ = Stmt\GroupUse[$3, $6, $2]; }
-    | T_USE legacy_namespace_name T_NS_SEPARATOR '{' inline_use_declarations '}'
-          { $$ = Stmt\GroupUse[$2, $5, Stmt\Use_::TYPE_UNKNOWN]; }
+      T_USE use_type namespace_name_parts T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
+          { $$ = Stmt\GroupUse[new Name($3, stackAttributes(#3)), $6, $2]; }
+    | T_USE use_type T_NS_SEPARATOR namespace_name_parts T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
+          { $$ = Stmt\GroupUse[new Name($4, stackAttributes(#4)), $7, $2]; }
+    | T_USE namespace_name_parts T_NS_SEPARATOR '{' inline_use_declarations '}'
+          { $$ = Stmt\GroupUse[new Name($2, stackAttributes(#2)), $5, Stmt\Use_::TYPE_UNKNOWN]; }
+    | T_USE T_NS_SEPARATOR namespace_name_parts T_NS_SEPARATOR '{' inline_use_declarations '}'
+          { $$ = Stmt\GroupUse[new Name($3, stackAttributes(#3)), $6, Stmt\Use_::TYPE_UNKNOWN]; }
 ;
 
 unprefixed_use_declarations:
@@ -128,10 +126,8 @@ unprefixed_use_declaration:
 ;
 
 use_declaration:
-      legacy_namespace_name
-          { $$ = Stmt\UseUse[$1, null, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #1); }
-    | legacy_namespace_name T_AS identifier
-          { $$ = Stmt\UseUse[$1, $3, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #3); }
+      unprefixed_use_declaration                            { $$ = $1; }
+    | T_NS_SEPARATOR unprefixed_use_declaration             { $$ = $2; }
 ;
 
 inline_use_declaration:
@@ -251,12 +247,7 @@ variables_list:
 
 optional_ref:
       /* empty */                                           { $$ = false; }
-    | ampersand                                             { $$ = true; }
-;
-
-optional_arg_ref:
-      /* empty */                                           { $$ = false; }
-    | T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG                 { $$ = true; }
+    | '&'                                                   { $$ = true; }
 ;
 
 optional_ellipsis:
@@ -388,7 +379,7 @@ new_else_single:
 
 foreach_variable:
       variable                                              { $$ = array($1, false); }
-    | ampersand variable                                    { $$ = array($2, true); }
+    | '&' variable                                          { $$ = array($2, true); }
     | list_expr                                             { $$ = array($1, false); }
 ;
 
@@ -403,9 +394,9 @@ non_empty_parameter_list:
 ;
 
 parameter:
-      optional_param_type optional_arg_ref optional_ellipsis plain_variable
+      optional_param_type optional_ref optional_ellipsis plain_variable
           { $$ = Node\Param[$4, null, $1, $2, $3]; $this->checkParam($$); }
-    | optional_param_type optional_arg_ref optional_ellipsis plain_variable '=' static_scalar
+    | optional_param_type optional_ref optional_ellipsis plain_variable '=' static_scalar
           { $$ = Node\Param[$4, $6, $1, $2, $3]; $this->checkParam($$); }
 ;
 
@@ -438,7 +429,7 @@ non_empty_argument_list:
 
 argument:
       expr                                                  { $$ = Node\Arg[$1, false, false]; }
-    | ampersand variable                                    { $$ = Node\Arg[$2, true, false]; }
+    | '&' variable                                          { $$ = Node\Arg[$2, true, false]; }
     | T_ELLIPSIS expr                                       { $$ = Node\Arg[$2, false, true]; }
 ;
 
@@ -572,8 +563,8 @@ expr:
       variable                                              { $$ = $1; }
     | list_expr '=' expr                                    { $$ = Expr\Assign[$1, $3]; }
     | variable '=' expr                                     { $$ = Expr\Assign[$1, $3]; }
-    | variable '=' ampersand variable                       { $$ = Expr\AssignRef[$1, $4]; }
-    | variable '=' ampersand new_expr                       { $$ = Expr\AssignRef[$1, $4]; }
+    | variable '=' '&' variable                             { $$ = Expr\AssignRef[$1, $4]; }
+    | variable '=' '&' new_expr                             { $$ = Expr\AssignRef[$1, $4]; }
     | new_expr                                              { $$ = $1; }
     | T_CLONE expr                                          { $$ = Expr\Clone_[$2]; }
     | variable T_PLUS_EQUAL expr                            { $$ = Expr\AssignOp\Plus      [$1, $3]; }
@@ -599,8 +590,7 @@ expr:
     | expr T_LOGICAL_AND expr                               { $$ = Expr\BinaryOp\LogicalAnd[$1, $3]; }
     | expr T_LOGICAL_XOR expr                               { $$ = Expr\BinaryOp\LogicalXor[$1, $3]; }
     | expr '|' expr                                         { $$ = Expr\BinaryOp\BitwiseOr [$1, $3]; }
-    | expr T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG expr   { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
-    | expr T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG expr       { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | expr '&' expr                                         { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
     | expr '^' expr                                         { $$ = Expr\BinaryOp\BitwiseXor[$1, $3]; }
     | expr '.' expr                                         { $$ = Expr\BinaryOp\Concat    [$1, $3]; }
     | expr '+' expr                                         { $$ = Expr\BinaryOp\Plus      [$1, $3]; }
@@ -743,10 +733,9 @@ class_name:
 ;
 
 name:
-      T_STRING                                              { $$ = Name[$1]; }
-    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
-    | T_NAME_FULLY_QUALIFIED                                { $$ = Name\FullyQualified[substr($1, 1)]; }
-    | T_NAME_RELATIVE                                       { $$ = Name\Relative[substr($1, 10)]; }
+      namespace_name_parts                                  { $$ = Name[$1]; }
+    | T_NS_SEPARATOR namespace_name_parts                   { $$ = Name\FullyQualified[$2]; }
+    | T_NAMESPACE T_NS_SEPARATOR namespace_name_parts       { $$ = Name\Relative[$3]; }
 ;
 
 class_name_reference:
@@ -827,10 +816,7 @@ static_operation:
     | static_scalar T_LOGICAL_AND static_scalar             { $$ = Expr\BinaryOp\LogicalAnd[$1, $3]; }
     | static_scalar T_LOGICAL_XOR static_scalar             { $$ = Expr\BinaryOp\LogicalXor[$1, $3]; }
     | static_scalar '|' static_scalar                       { $$ = Expr\BinaryOp\BitwiseOr [$1, $3]; }
-    | static_scalar T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG static_scalar
-          { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
-    | static_scalar T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG static_scalar
-          { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | static_scalar '&' static_scalar                       { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
     | static_scalar '^' static_scalar                       { $$ = Expr\BinaryOp\BitwiseXor[$1, $3]; }
     | static_scalar '.' static_scalar                       { $$ = Expr\BinaryOp\Concat    [$1, $3]; }
     | static_scalar '+' static_scalar                       { $$ = Expr\BinaryOp\Plus      [$1, $3]; }
@@ -1000,8 +986,8 @@ non_empty_array_pair_list:
 array_pair:
       expr T_DOUBLE_ARROW expr                              { $$ = Expr\ArrayItem[$3, $1,   false]; }
     | expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
-    | expr T_DOUBLE_ARROW ampersand variable                { $$ = Expr\ArrayItem[$4, $1,   true]; }
-    | ampersand variable                                    { $$ = Expr\ArrayItem[$2, null, true]; }
+    | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
+    | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
     | T_ELLIPSIS expr                                       { $$ = Expr\ArrayItem[$2, null, false, attributes(), true]; }
 ;
 

grammar/php7.y

@@ -20,11 +20,6 @@ top_statement_list:
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
-ampersand:
-      T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
-    | T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG
-;
-
 reserved_non_modifiers:
       T_INCLUDE | T_INCLUDE_ONCE | T_EVAL | T_REQUIRE | T_REQUIRE_ONCE | T_LOGICAL_OR | T_LOGICAL_XOR | T_LOGICAL_AND
     | T_INSTANCEOF | T_NEW | T_CLONE | T_EXIT | T_IF | T_ELSEIF | T_ELSE | T_ENDIF | T_ECHO | T_DO | T_WHILE
@@ -33,20 +28,19 @@ reserved_non_modifiers:
     | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT
     | T_BREAK | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE | T_CLASS
     | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER | T_FN
-    | T_MATCH | T_ENUM
 ;
 
 semi_reserved:
       reserved_non_modifiers
-    | T_STATIC | T_ABSTRACT | T_FINAL | T_PRIVATE | T_PROTECTED | T_PUBLIC | T_READONLY
+    | T_STATIC | T_ABSTRACT | T_FINAL | T_PRIVATE | T_PROTECTED | T_PUBLIC
 ;
 
-identifier_maybe_reserved:
+identifier_ex:
       T_STRING                                              { $$ = Node\Identifier[$1]; }
     | semi_reserved                                         { $$ = Node\Identifier[$1]; }
 ;
 
-identifier_not_reserved:
+identifier:
       T_STRING                                              { $$ = Node\Identifier[$1]; }
 ;
 
@@ -54,20 +48,13 @@ reserved_non_modifiers_identifier:
       reserved_non_modifiers                                { $$ = Node\Identifier[$1]; }
 ;
 
-namespace_declaration_name:
-      T_STRING                                              { $$ = Name[$1]; }
-    | semi_reserved                                         { $$ = Name[$1]; }
-    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
+namespace_name_parts:
+      T_STRING                                              { init($1); }
+    | namespace_name_parts T_NS_SEPARATOR T_STRING          { push($1, $3); }
 ;
 
 namespace_name:
-      T_STRING                                              { $$ = Name[$1]; }
-    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
-;
-
-legacy_namespace_name:
-      namespace_name                                        { $$ = $1; }
-    | T_NAME_FULLY_QUALIFIED                                { $$ = Name[substr($1, 1)]; }
+      namespace_name_parts                                  { $$ = Name[$1]; }
 ;
 
 plain_variable:
@@ -87,31 +74,6 @@ no_comma:
 optional_comma:
       /* empty */
     | ','
-;
-
-attribute_decl:
-      class_name                                            { $$ = Node\Attribute[$1, []]; }
-    | class_name argument_list                              { $$ = Node\Attribute[$1, $2]; }
-;
-
-attribute_group:
-      attribute_decl                                        { init($1); }
-    | attribute_group ',' attribute_decl                    { push($1, $3); }
-;
-
-attribute:
-      T_ATTRIBUTE attribute_group optional_comma ']'        { $$ = Node\AttributeGroup[$2]; }
-;
-
-attributes:
-      attribute                                             { init($1); }
-    | attributes attribute                                  { push($1, $2); }
-;
-
-optional_attributes:
-      /* empty */                                           { $$ = []; }
-    | attributes                                            { $$ = $1; }
-;
 
 top_statement:
       statement                                             { $$ = $1; }
@@ -119,11 +81,11 @@ top_statement:
     | class_declaration_statement                           { $$ = $1; }
     | T_HALT_COMPILER
           { $$ = Stmt\HaltCompiler[$this->lexer->handleHaltCompiler()]; }
-    | T_NAMESPACE namespace_declaration_name semi
+    | T_NAMESPACE namespace_name semi
           { $$ = Stmt\Namespace_[$2, null];
             $$->setAttribute('kind', Stmt\Namespace_::KIND_SEMICOLON);
             $this->checkNamespace($$); }
-    | T_NAMESPACE namespace_declaration_name '{' top_statement_list '}'
+    | T_NAMESPACE namespace_name '{' top_statement_list '}'
           { $$ = Stmt\Namespace_[$2, $4];
             $$->setAttribute('kind', Stmt\Namespace_::KIND_BRACED);
             $this->checkNamespace($$); }
@@ -142,11 +104,16 @@ use_type:
     | T_CONST                                               { $$ = Stmt\Use_::TYPE_CONSTANT; }
 ;
 
+/* Using namespace_name_parts here to avoid s/r conflict on T_NS_SEPARATOR */
 group_use_declaration:
-      T_USE use_type legacy_namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
-          { $$ = Stmt\GroupUse[$3, $6, $2]; }
-    | T_USE legacy_namespace_name T_NS_SEPARATOR '{' inline_use_declarations '}'
-          { $$ = Stmt\GroupUse[$2, $5, Stmt\Use_::TYPE_UNKNOWN]; }
+      T_USE use_type namespace_name_parts T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
+          { $$ = Stmt\GroupUse[new Name($3, stackAttributes(#3)), $6, $2]; }
+    | T_USE use_type T_NS_SEPARATOR namespace_name_parts T_NS_SEPARATOR '{' unprefixed_use_declarations '}'
+          { $$ = Stmt\GroupUse[new Name($4, stackAttributes(#4)), $7, $2]; }
+    | T_USE namespace_name_parts T_NS_SEPARATOR '{' inline_use_declarations '}'
+          { $$ = Stmt\GroupUse[new Name($2, stackAttributes(#2)), $5, Stmt\Use_::TYPE_UNKNOWN]; }
+    | T_USE T_NS_SEPARATOR namespace_name_parts T_NS_SEPARATOR '{' inline_use_declarations '}'
+          { $$ = Stmt\GroupUse[new Name($3, stackAttributes(#3)), $6, Stmt\Use_::TYPE_UNKNOWN]; }
 ;
 
 unprefixed_use_declarations:
@@ -181,15 +148,13 @@ non_empty_inline_use_declarations:
 unprefixed_use_declaration:
       namespace_name
           { $$ = Stmt\UseUse[$1, null, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #1); }
-    | namespace_name T_AS identifier_not_reserved
+    | namespace_name T_AS identifier
           { $$ = Stmt\UseUse[$1, $3, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #3); }
 ;
 
 use_declaration:
-      legacy_namespace_name
-          { $$ = Stmt\UseUse[$1, null, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #1); }
-    | legacy_namespace_name T_AS identifier_not_reserved
-          { $$ = Stmt\UseUse[$1, $3, Stmt\Use_::TYPE_UNKNOWN]; $this->checkUseUse($$, #3); }
+      unprefixed_use_declaration                            { $$ = $1; }
+    | T_NS_SEPARATOR unprefixed_use_declaration             { $$ = $2; }
 ;
 
 inline_use_declaration:
@@ -208,7 +173,7 @@ non_empty_constant_declaration_list:
 ;
 
 constant_declaration:
-    identifier_not_reserved '=' expr                        { $$ = Node\Const_[$1, $3]; }
+    identifier '=' expr                                     { $$ = Node\Const_[$1, $3]; }
 ;
 
 class_const_list:
@@ -221,7 +186,7 @@ non_empty_class_const_list:
 ;
 
 class_const:
-    identifier_maybe_reserved '=' expr                      { $$ = Node\Const_[$1, $3]; }
+    identifier_ex '=' expr                                  { $$ = Node\Const_[$1, $3]; }
 ;
 
 inner_statement_list_ex:
@@ -267,18 +232,9 @@ non_empty_statement:
     | T_RETURN optional_expr semi                           { $$ = Stmt\Return_[$2]; }
     | T_GLOBAL global_var_list semi                         { $$ = Stmt\Global_[$2]; }
     | T_STATIC static_var_list semi                         { $$ = Stmt\Static_[$2]; }
-    | T_ECHO expr_list_forbid_comma semi                    { $$ = Stmt\Echo_[$2]; }
+    | T_ECHO expr_list semi                                 { $$ = Stmt\Echo_[$2]; }
     | T_INLINE_HTML                                         { $$ = Stmt\InlineHTML[$1]; }
-    | expr semi {
-        $e = $1;
-        if ($e instanceof Expr\Throw_) {
-            // For backwards-compatibility reasons, convert throw in statement position into
-            // Stmt\Throw_ rather than Stmt\Expression(Expr\Throw_).
-            $$ = Stmt\Throw_[$e->expr];
-        } else {
-            $$ = Stmt\Expression[$e];
-        }
-    }
+    | expr semi                                             { $$ = Stmt\Expression[$1]; }
     | T_UNSET '(' variables_list ')' semi                   { $$ = Stmt\Unset_[$3]; }
     | T_FOREACH '(' expr T_AS foreach_variable ')' foreach_statement
           { $$ = Stmt\Foreach_[$3, $5[0], ['keyVar' => null, 'byRef' => $5[1], 'stmts' => $7]]; }
@@ -289,8 +245,9 @@ non_empty_statement:
     | T_DECLARE '(' declare_list ')' declare_statement      { $$ = Stmt\Declare_[$3, $5]; }
     | T_TRY '{' inner_statement_list '}' catches optional_finally
           { $$ = Stmt\TryCatch[$3, $5, $6]; $this->checkTryCatch($$); }
-    | T_GOTO identifier_not_reserved semi                   { $$ = Stmt\Goto_[$2]; }
-    | identifier_not_reserved ':'                           { $$ = Stmt\Label[$1]; }
+    | T_THROW expr semi                                     { $$ = Stmt\Throw_[$2]; }
+    | T_GOTO identifier semi                                { $$ = Stmt\Goto_[$2]; }
+    | identifier ':'                                        { $$ = Stmt\Label[$1]; }
     | error                                                 { $$ = array(); /* means: no statement */ }
 ;
 
@@ -312,7 +269,7 @@ name_union:
 ;
 
 catch:
-    T_CATCH '(' name_union optional_plain_variable ')' '{' inner_statement_list '}'
+    T_CATCH '(' name_union plain_variable ')' '{' inner_statement_list '}'
         { $$ = Stmt\Catch_[$3, $4, $7]; }
 ;
 
@@ -332,12 +289,7 @@ non_empty_variables_list:
 
 optional_ref:
       /* empty */                                           { $$ = false; }
-    | ampersand                                             { $$ = true; }
-;
-
-optional_arg_ref:
-      /* empty */                                           { $$ = false; }
-    | T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG                 { $$ = true; }
+    | '&'                                                   { $$ = true; }
 ;
 
 optional_ellipsis:
@@ -351,33 +303,19 @@ block_or_error:
 ;
 
 function_declaration_statement:
-      T_FUNCTION optional_ref identifier_not_reserved '(' parameter_list ')' optional_return_type block_or_error
-          { $$ = Stmt\Function_[$3, ['byRef' => $2, 'params' => $5, 'returnType' => $7, 'stmts' => $8, 'attrGroups' => []]]; }
-    | attributes T_FUNCTION optional_ref identifier_not_reserved '(' parameter_list ')' optional_return_type block_or_error
-          { $$ = Stmt\Function_[$4, ['byRef' => $3, 'params' => $6, 'returnType' => $8, 'stmts' => $9, 'attrGroups' => $1]]; }
+    T_FUNCTION optional_ref identifier '(' parameter_list ')' optional_return_type block_or_error
+        { $$ = Stmt\Function_[$3, ['byRef' => $2, 'params' => $5, 'returnType' => $7, 'stmts' => $8]]; }
 ;
 
 class_declaration_statement:
-      optional_attributes class_entry_type identifier_not_reserved extends_from implements_list '{' class_statement_list '}'
-          { $$ = Stmt\Class_[$3, ['type' => $2, 'extends' => $4, 'implements' => $5, 'stmts' => $7, 'attrGroups' => $1]];
-            $this->checkClass($$, #3); }
-    | optional_attributes T_INTERFACE identifier_not_reserved interface_extends_list '{' class_statement_list '}'
-          { $$ = Stmt\Interface_[$3, ['extends' => $4, 'stmts' => $6, 'attrGroups' => $1]];
-            $this->checkInterface($$, #3); }
-    | optional_attributes T_TRAIT identifier_not_reserved '{' class_statement_list '}'
-          { $$ = Stmt\Trait_[$3, ['stmts' => $5, 'attrGroups' => $1]]; }
-    | optional_attributes T_ENUM identifier_not_reserved enum_scalar_type implements_list '{' class_statement_list '}'
-          { $$ = Stmt\Enum_[$3, ['scalarType' => $4, 'implements' => $5, 'stmts' => $7, 'attrGroups' => $1]];
-            $this->checkEnum($$, #3); }
-;
-
-enum_scalar_type:
-      /* empty */                                           { $$ = null; }
-    | ':' type                                              { $$ = $2; }
-
-enum_case_expr:
-      /* empty */                                           { $$ = null; }
-    | '=' expr                                              { $$ = $2; }
+      class_entry_type identifier extends_from implements_list '{' class_statement_list '}'
+          { $$ = Stmt\Class_[$2, ['type' => $1, 'extends' => $3, 'implements' => $4, 'stmts' => $6]];
+            $this->checkClass($$, #2); }
+    | T_INTERFACE identifier interface_extends_list '{' class_statement_list '}'
+          { $$ = Stmt\Interface_[$2, ['extends' => $3, 'stmts' => $5]];
+            $this->checkInterface($$, #2); }
+    | T_TRAIT identifier '{' class_statement_list '}'
+          { $$ = Stmt\Trait_[$2, ['stmts' => $4]]; }
 ;
 
 class_entry_type:
@@ -436,7 +374,7 @@ non_empty_declare_list:
 ;
 
 declare_list_element:
-      identifier_not_reserved '=' expr                      { $$ = Stmt\DeclareDeclare[$1, $3]; }
+      identifier '=' expr                                   { $$ = Stmt\DeclareDeclare[$1, $3]; }
 ;
 
 switch_case_list:
@@ -461,25 +399,6 @@ case_separator:
     | ';'
 ;
 
-match:
-      T_MATCH '(' expr ')' '{' match_arm_list '}'           { $$ = Expr\Match_[$3, $6]; }
-;
-
-match_arm_list:
-      /* empty */                                           { $$ = []; }
-    | non_empty_match_arm_list optional_comma               { $$ = $1; }
-;
-
-non_empty_match_arm_list:
-      match_arm                                             { init($1); }
-    | non_empty_match_arm_list ',' match_arm                { push($1, $3); }
-;
-
-match_arm:
-      expr_list_allow_comma T_DOUBLE_ARROW expr             { $$ = Node\MatchArm[$1, $3]; }
-    | T_DEFAULT optional_comma T_DOUBLE_ARROW expr          { $$ = Node\MatchArm[null, $4]; }
-;
-
 while_statement:
       statement                                             { $$ = toArray($1); }
     | ':' inner_statement_list T_ENDWHILE ';'               { $$ = $2; }
@@ -515,13 +434,13 @@ new_else_single:
 
 foreach_variable:
       variable                                              { $$ = array($1, false); }
-    | ampersand variable                                    { $$ = array($2, true); }
+    | '&' variable                                          { $$ = array($2, true); }
     | list_expr                                             { $$ = array($1, false); }
     | array_short_syntax                                    { $$ = array($1, false); }
 ;
 
 parameter_list:
-      non_empty_parameter_list optional_comma               { $$ = $1; }
+      non_empty_parameter_list no_comma                     { $$ = $1; }
     | /* empty */                                           { $$ = array(); }
 ;
 
@@ -530,84 +449,29 @@ non_empty_parameter_list:
     | non_empty_parameter_list ',' parameter                { push($1, $3); }
 ;
 
-optional_property_modifiers:
-      /* empty */               { $$ = 0; }
-    | optional_property_modifiers property_modifier
-          { $this->checkModifier($1, $2, #2); $$ = $1 | $2; }
-;
-
-property_modifier:
-      T_PUBLIC                  { $$ = Stmt\Class_::MODIFIER_PUBLIC; }
-    | T_PROTECTED               { $$ = Stmt\Class_::MODIFIER_PROTECTED; }
-    | T_PRIVATE                 { $$ = Stmt\Class_::MODIFIER_PRIVATE; }
-    | T_READONLY                { $$ = Stmt\Class_::MODIFIER_READONLY; }
-;
-
 parameter:
-      optional_attributes optional_property_modifiers optional_type_without_static
-      optional_arg_ref optional_ellipsis plain_variable
-          { $$ = new Node\Param($6, null, $3, $4, $5, attributes(), $2, $1);
-            $this->checkParam($$); }
-    | optional_attributes optional_property_modifiers optional_type_without_static
-      optional_arg_ref optional_ellipsis plain_variable '=' expr
-          { $$ = new Node\Param($6, $8, $3, $4, $5, attributes(), $2, $1);
-            $this->checkParam($$); }
-    | optional_attributes optional_property_modifiers optional_type_without_static
-      optional_arg_ref optional_ellipsis error
-          { $$ = new Node\Param(Expr\Error[], null, $3, $4, $5, attributes(), $2, $1); }
+      optional_type optional_ref optional_ellipsis plain_variable
+          { $$ = Node\Param[$4, null, $1, $2, $3]; $this->checkParam($$); }
+    | optional_type optional_ref optional_ellipsis plain_variable '=' expr
+          { $$ = Node\Param[$4, $6, $1, $2, $3]; $this->checkParam($$); }
+    | optional_type optional_ref optional_ellipsis error
+          { $$ = Node\Param[Expr\Error[], null, $1, $2, $3]; }
 ;
 
 type_expr:
       type                                                  { $$ = $1; }
     | '?' type                                              { $$ = Node\NullableType[$2]; }
-    | union_type                                            { $$ = Node\UnionType[$1]; }
-    | intersection_type                                     { $$ = Node\IntersectionType[$1]; }
 ;
 
 type:
-      type_without_static                                   { $$ = $1; }
-    | T_STATIC                                              { $$ = Node\Name['static']; }
-;
-
-type_without_static:
       name                                                  { $$ = $this->handleBuiltinTypes($1); }
     | T_ARRAY                                               { $$ = Node\Identifier['array']; }
     | T_CALLABLE                                            { $$ = Node\Identifier['callable']; }
 ;
 
-union_type:
-      type '|' type                                         { init($1, $3); }
-    | union_type '|' type                                   { push($1, $3); }
-;
-
-union_type_without_static:
-      type_without_static '|' type_without_static           { init($1, $3); }
-    | union_type_without_static '|' type_without_static     { push($1, $3); }
-;
-
-intersection_type:
-      type T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG type   { init($1, $3); }
-    | intersection_type T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG type
-          { push($1, $3); }
-;
-
-intersection_type_without_static:
-      type_without_static T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG type_without_static
-          { init($1, $3); }
-    | intersection_type_without_static T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG type_without_static
-          { push($1, $3); }
-;
-
-type_expr_without_static:
-      type_without_static                                   { $$ = $1; }
-    | '?' type_without_static                               { $$ = Node\NullableType[$2]; }
-    | union_type_without_static                             { $$ = Node\UnionType[$1]; }
-    | intersection_type_without_static                      { $$ = Node\IntersectionType[$1]; }
-;
-
-optional_type_without_static:
+optional_type:
       /* empty */                                           { $$ = null; }
-    | type_expr_without_static                              { $$ = $1; }
+    | type_expr                                             { $$ = $1; }
 ;
 
 optional_return_type:
@@ -619,11 +483,6 @@ optional_return_type:
 argument_list:
       '(' ')'                                               { $$ = array(); }
     | '(' non_empty_argument_list optional_comma ')'        { $$ = $2; }
-    | '(' variadic_placeholder ')'                          { init($2); }
-;
-
-variadic_placeholder:
-      T_ELLIPSIS                                            { $$ = Node\VariadicPlaceholder[]; }
 ;
 
 non_empty_argument_list:
@@ -633,10 +492,8 @@ non_empty_argument_list:
 
 argument:
       expr                                                  { $$ = Node\Arg[$1, false, false]; }
-    | ampersand variable                                    { $$ = Node\Arg[$2, true, false]; }
+    | '&' variable                                          { $$ = Node\Arg[$2, true, false]; }
     | T_ELLIPSIS expr                                       { $$ = Node\Arg[$2, false, true]; }
-    | identifier_maybe_reserved ':' expr
-          { $$ = new Node\Arg($3, false, false, attributes(), $1); }
 ;
 
 global_var_list:
@@ -649,7 +506,7 @@ non_empty_global_var_list:
 ;
 
 global_var:
-      simple_variable                                       { $$ = $1; }
+      simple_variable                                       { $$ = Expr\Variable[$1]; }
 ;
 
 static_var_list:
@@ -678,19 +535,15 @@ class_statement_list:
 ;
 
 class_statement:
-      optional_attributes variable_modifiers optional_type_without_static property_declaration_list semi
-          { $$ = new Stmt\Property($2, $4, attributes(), $3, $1);
-            $this->checkProperty($$, #2); }
-    | optional_attributes method_modifiers T_CONST class_const_list semi
-          { $$ = new Stmt\ClassConst($4, $2, attributes(), $1);
-            $this->checkClassConst($$, #2); }
-    | optional_attributes method_modifiers T_FUNCTION optional_ref identifier_maybe_reserved '(' parameter_list ')'
-      optional_return_type method_body
-          { $$ = Stmt\ClassMethod[$5, ['type' => $2, 'byRef' => $4, 'params' => $7, 'returnType' => $9, 'stmts' => $10, 'attrGroups' => $1]];
-            $this->checkClassMethod($$, #2); }
+      variable_modifiers optional_type property_declaration_list ';'
+          { $attrs = attributes();
+            $$ = new Stmt\Property($1, $3, $attrs, $2); $this->checkProperty($$, #1); }
+    | method_modifiers T_CONST class_const_list ';'
+          { $$ = Stmt\ClassConst[$3, $1]; $this->checkClassConst($$, #1); }
+    | method_modifiers T_FUNCTION optional_ref identifier_ex '(' parameter_list ')' optional_return_type method_body
+          { $$ = Stmt\ClassMethod[$4, ['type' => $1, 'byRef' => $3, 'params' => $6, 'returnType' => $8, 'stmts' => $9]];
+            $this->checkClassMethod($$, #1); }
     | T_USE class_name_list trait_adaptations               { $$ = Stmt\TraitUse[$2, $3]; }
-    | optional_attributes T_CASE identifier_maybe_reserved enum_case_expr semi
-         { $$ = Stmt\EnumCase[$3, $4, $1]; }
     | error                                                 { $$ = null; /* will be skipped */ }
 ;
 
@@ -707,22 +560,22 @@ trait_adaptation_list:
 trait_adaptation:
       trait_method_reference_fully_qualified T_INSTEADOF class_name_list ';'
           { $$ = Stmt\TraitUseAdaptation\Precedence[$1[0], $1[1], $3]; }
-    | trait_method_reference T_AS member_modifier identifier_maybe_reserved ';'
+    | trait_method_reference T_AS member_modifier identifier_ex ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], $3, $4]; }
     | trait_method_reference T_AS member_modifier ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], $3, null]; }
-    | trait_method_reference T_AS identifier_not_reserved ';'
+    | trait_method_reference T_AS identifier ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], null, $3]; }
     | trait_method_reference T_AS reserved_non_modifiers_identifier ';'
           { $$ = Stmt\TraitUseAdaptation\Alias[$1[0], $1[1], null, $3]; }
 ;
 
 trait_method_reference_fully_qualified:
-      name T_PAAMAYIM_NEKUDOTAYIM identifier_maybe_reserved { $$ = array($1, $3); }
+      name T_PAAMAYIM_NEKUDOTAYIM identifier_ex             { $$ = array($1, $3); }
 ;
 trait_method_reference:
       trait_method_reference_fully_qualified                { $$ = $1; }
-    | identifier_maybe_reserved                             { $$ = array(null, $1); }
+    | identifier_ex                                         { $$ = array(null, $1); }
 ;
 
 method_body:
@@ -752,7 +605,6 @@ member_modifier:
     | T_STATIC                                              { $$ = Stmt\Class_::MODIFIER_STATIC; }
     | T_ABSTRACT                                            { $$ = Stmt\Class_::MODIFIER_ABSTRACT; }
     | T_FINAL                                               { $$ = Stmt\Class_::MODIFIER_FINAL; }
-    | T_READONLY                                            { $$ = Stmt\Class_::MODIFIER_READONLY; }
 ;
 
 property_declaration_list:
@@ -774,14 +626,10 @@ property_declaration:
     | property_decl_name '=' expr                           { $$ = Stmt\PropertyProperty[$1, $3]; }
 ;
 
-expr_list_forbid_comma:
+expr_list:
       non_empty_expr_list no_comma                          { $$ = $1; }
 ;
 
-expr_list_allow_comma:
-      non_empty_expr_list optional_comma                    { $$ = $1; }
-;
-
 non_empty_expr_list:
       non_empty_expr_list ',' expr                          { push($1, $3); }
     | expr                                                  { init($1); }
@@ -789,7 +637,7 @@ non_empty_expr_list:
 
 for_expr:
       /* empty */                                           { $$ = array(); }
-    | expr_list_forbid_comma                                { $$ = $1; }
+    | expr_list                                             { $$ = $1; }
 ;
 
 expr:
@@ -797,9 +645,8 @@ expr:
     | list_expr '=' expr                                    { $$ = Expr\Assign[$1, $3]; }
     | array_short_syntax '=' expr                           { $$ = Expr\Assign[$1, $3]; }
     | variable '=' expr                                     { $$ = Expr\Assign[$1, $3]; }
-    | variable '=' ampersand variable                       { $$ = Expr\AssignRef[$1, $4]; }
+    | variable '=' '&' variable                             { $$ = Expr\AssignRef[$1, $4]; }
     | new_expr                                              { $$ = $1; }
-    | match                                                 { $$ = $1; }
     | T_CLONE expr                                          { $$ = Expr\Clone_[$2]; }
     | variable T_PLUS_EQUAL expr                            { $$ = Expr\AssignOp\Plus      [$1, $3]; }
     | variable T_MINUS_EQUAL expr                           { $$ = Expr\AssignOp\Minus     [$1, $3]; }
@@ -824,8 +671,7 @@ expr:
     | expr T_LOGICAL_AND expr                               { $$ = Expr\BinaryOp\LogicalAnd[$1, $3]; }
     | expr T_LOGICAL_XOR expr                               { $$ = Expr\BinaryOp\LogicalXor[$1, $3]; }
     | expr '|' expr                                         { $$ = Expr\BinaryOp\BitwiseOr [$1, $3]; }
-    | expr T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG expr   { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
-    | expr T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG expr       { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
+    | expr '&' expr                                         { $$ = Expr\BinaryOp\BitwiseAnd[$1, $3]; }
     | expr '^' expr                                         { $$ = Expr\BinaryOp\BitwiseXor[$1, $3]; }
     | expr '.' expr                                         { $$ = Expr\BinaryOp\Concat    [$1, $3]; }
     | expr '+' expr                                         { $$ = Expr\BinaryOp\Plus      [$1, $3]; }
@@ -854,7 +700,7 @@ expr:
     | expr '?' expr ':' expr                                { $$ = Expr\Ternary[$1, $3,   $5]; }
     | expr '?' ':' expr                                     { $$ = Expr\Ternary[$1, null, $4]; }
     | expr T_COALESCE expr                                  { $$ = Expr\BinaryOp\Coalesce[$1, $3]; }
-    | T_ISSET '(' expr_list_allow_comma ')'                 { $$ = Expr\Isset_[$3]; }
+    | T_ISSET '(' variables_list ')'                        { $$ = Expr\Isset_[$3]; }
     | T_EMPTY '(' expr ')'                                  { $$ = Expr\Empty_[$3]; }
     | T_INCLUDE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_INCLUDE]; }
     | T_INCLUDE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_INCLUDE_ONCE]; }
@@ -883,30 +729,23 @@ expr:
     | T_YIELD expr                                          { $$ = Expr\Yield_[$2, null]; }
     | T_YIELD expr T_DOUBLE_ARROW expr                      { $$ = Expr\Yield_[$4, $2]; }
     | T_YIELD_FROM expr                                     { $$ = Expr\YieldFrom[$2]; }
-    | T_THROW expr                                          { $$ = Expr\Throw_[$2]; }
 
-    | T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr %prec T_THROW
-          { $$ = Expr\ArrowFunction[['static' => false, 'byRef' => $2, 'params' => $4, 'returnType' => $6, 'expr' => $8, 'attrGroups' => []]]; }
-    | T_STATIC T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr %prec T_THROW
-          { $$ = Expr\ArrowFunction[['static' => true, 'byRef' => $3, 'params' => $5, 'returnType' => $7, 'expr' => $9, 'attrGroups' => []]]; }
-    | T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type block_or_error
-          { $$ = Expr\Closure[['static' => false, 'byRef' => $2, 'params' => $4, 'uses' => $6, 'returnType' => $7, 'stmts' => $8, 'attrGroups' => []]]; }
-    | T_STATIC T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type       block_or_error
-          { $$ = Expr\Closure[['static' => true, 'byRef' => $3, 'params' => $5, 'uses' => $7, 'returnType' => $8, 'stmts' => $9, 'attrGroups' => []]]; }
+    | T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr
+          { $$ = Expr\ArrowFunction[['static' => false, 'byRef' => $2, 'params' => $4, 'returnType' => $6, 'expr' => $8]]; }
+    | T_STATIC T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr
+          { $$ = Expr\ArrowFunction[['static' => true, 'byRef' => $3, 'params' => $5, 'returnType' => $7, 'expr' => $9]]; }
 
-    | attributes T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr %prec T_THROW
-          { $$ = Expr\ArrowFunction[['static' => false, 'byRef' => $3, 'params' => $5, 'returnType' => $7, 'expr' => $9, 'attrGroups' => $1]]; }
-    | attributes T_STATIC T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr %prec T_THROW
-          { $$ = Expr\ArrowFunction[['static' => true, 'byRef' => $4, 'params' => $6, 'returnType' => $8, 'expr' => $10, 'attrGroups' => $1]]; }
-    | attributes T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type block_or_error
-          { $$ = Expr\Closure[['static' => false, 'byRef' => $3, 'params' => $5, 'uses' => $7, 'returnType' => $8, 'stmts' => $9, 'attrGroups' => $1]]; }
-    | attributes T_STATIC T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type       block_or_error
-          { $$ = Expr\Closure[['static' => true, 'byRef' => $4, 'params' => $6, 'uses' => $8, 'returnType' => $9, 'stmts' => $10, 'attrGroups' => $1]]; }
+    | T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type
+      block_or_error
+          { $$ = Expr\Closure[['static' => false, 'byRef' => $2, 'params' => $4, 'uses' => $6, 'returnType' => $7, 'stmts' => $8]]; }
+    | T_STATIC T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type
+      block_or_error
+          { $$ = Expr\Closure[['static' => true, 'byRef' => $3, 'params' => $5, 'uses' => $7, 'returnType' => $8, 'stmts' => $9]]; }
 ;
 
 anonymous_class:
-      optional_attributes T_CLASS ctor_arguments extends_from implements_list '{' class_statement_list '}'
-          { $$ = array(Stmt\Class_[null, ['type' => 0, 'extends' => $4, 'implements' => $5, 'stmts' => $7, 'attrGroups' => $1]], $3);
+      T_CLASS ctor_arguments extends_from implements_list '{' class_statement_list '}'
+          { $$ = array(Stmt\Class_[null, ['type' => 0, 'extends' => $3, 'implements' => $4, 'stmts' => $6]], $2);
             $this->checkClass($$[0], -1); }
 ;
 
@@ -922,7 +761,7 @@ lexical_vars:
 ;
 
 lexical_var_list:
-      non_empty_lexical_var_list optional_comma             { $$ = $1; }
+      non_empty_lexical_var_list no_comma                   { $$ = $1; }
 ;
 
 non_empty_lexical_var_list:
@@ -947,22 +786,20 @@ class_name:
 ;
 
 name:
-      T_STRING                                              { $$ = Name[$1]; }
-    | T_NAME_QUALIFIED                                      { $$ = Name[$1]; }
-    | T_NAME_FULLY_QUALIFIED                                { $$ = Name\FullyQualified[substr($1, 1)]; }
-    | T_NAME_RELATIVE                                       { $$ = Name\Relative[substr($1, 10)]; }
+      namespace_name_parts                                  { $$ = Name[$1]; }
+    | T_NS_SEPARATOR namespace_name_parts                   { $$ = Name\FullyQualified[$2]; }
+    | T_NAMESPACE T_NS_SEPARATOR namespace_name_parts       { $$ = Name\Relative[$3]; }
 ;
 
 class_name_reference:
       class_name                                            { $$ = $1; }
     | new_variable                                          { $$ = $1; }
-    | '(' expr ')'                                          { $$ = $2; }
     | error                                                 { $$ = Expr\Error[]; $this->errorState = 2; }
 ;
 
 class_name_or_var:
       class_name                                            { $$ = $1; }
-    | fully_dereferencable                                  { $$ = $1; }
+    | dereferencable                                        { $$ = $1; }
 ;
 
 exit_expr:
@@ -984,20 +821,9 @@ ctor_arguments:
 
 constant:
       name                                                  { $$ = Expr\ConstFetch[$1]; }
-    | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }
-    | T_FILE                                                { $$ = Scalar\MagicConst\File[]; }
-    | T_DIR                                                 { $$ = Scalar\MagicConst\Dir[]; }
-    | T_CLASS_C                                             { $$ = Scalar\MagicConst\Class_[]; }
-    | T_TRAIT_C                                             { $$ = Scalar\MagicConst\Trait_[]; }
-    | T_METHOD_C                                            { $$ = Scalar\MagicConst\Method[]; }
-    | T_FUNC_C                                              { $$ = Scalar\MagicConst\Function_[]; }
-    | T_NS_C                                                { $$ = Scalar\MagicConst\Namespace_[]; }
-;
-
-class_constant:
-      class_name_or_var T_PAAMAYIM_NEKUDOTAYIM identifier_maybe_reserved
+    | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM identifier_ex
           { $$ = Expr\ClassConstFetch[$1, $3]; }
-    /* We interpret an isolated FOO:: as an unfinished class constant fetch. It could also be
+    /* We interpret and isolated FOO:: as an unfinished class constant fetch. It could also be
        an unfinished static property fetch or unfinished scoped call. */
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM error
           { $$ = Expr\ClassConstFetch[$1, new Expr\Error(stackAttributes(#3))]; $this->errorState = 2; }
@@ -1017,21 +843,28 @@ dereferencable_scalar:
     | T_CONSTANT_ENCAPSED_STRING
           { $attrs = attributes(); $attrs['kind'] = strKind($1);
             $$ = new Scalar\String_(Scalar\String_::parse($1), $attrs); }
-    | '"' encaps_list '"'
-          { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
-            parseEncapsed($2, '"', true); $$ = new Scalar\Encapsed($2, $attrs); }
 ;
 
 scalar:
       T_LNUMBER                                             { $$ = $this->parseLNumber($1, attributes()); }
     | T_DNUMBER                                             { $$ = Scalar\DNumber[Scalar\DNumber::parse($1)]; }
+    | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }
+    | T_FILE                                                { $$ = Scalar\MagicConst\File[]; }
+    | T_DIR                                                 { $$ = Scalar\MagicConst\Dir[]; }
+    | T_CLASS_C                                             { $$ = Scalar\MagicConst\Class_[]; }
+    | T_TRAIT_C                                             { $$ = Scalar\MagicConst\Trait_[]; }
+    | T_METHOD_C                                            { $$ = Scalar\MagicConst\Method[]; }
+    | T_FUNC_C                                              { $$ = Scalar\MagicConst\Function_[]; }
+    | T_NS_C                                                { $$ = Scalar\MagicConst\Namespace_[]; }
     | dereferencable_scalar                                 { $$ = $1; }
     | constant                                              { $$ = $1; }
-    | class_constant                                        { $$ = $1; }
     | T_START_HEREDOC T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC
           { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), true); }
     | T_START_HEREDOC T_END_HEREDOC
           { $$ = $this->parseDocString($1, '', $2, attributes(), stackAttributes(#2), true); }
+    | '"' encaps_list '"'
+          { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
+            parseEncapsed($2, '"', true); $$ = new Scalar\Encapsed($2, $attrs); }
     | T_START_HEREDOC encaps_list T_END_HEREDOC
           { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), true); }
 ;
@@ -1041,16 +874,10 @@ optional_expr:
     | expr                                                  { $$ = $1; }
 ;
 
-fully_dereferencable:
+dereferencable:
       variable                                              { $$ = $1; }
     | '(' expr ')'                                          { $$ = $2; }
     | dereferencable_scalar                                 { $$ = $1; }
-    | class_constant                                        { $$ = $1; }
-;
-
-array_object_dereferencable:
-      fully_dereferencable                                  { $$ = $1; }
-    | constant                                              { $$ = $1; }
 ;
 
 callable_expr:
@@ -1060,40 +887,31 @@ callable_expr:
 ;
 
 callable_variable:
-      simple_variable                                       { $$ = $1; }
-    | array_object_dereferencable '[' optional_expr ']'     { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | array_object_dereferencable '{' expr '}'              { $$ = Expr\ArrayDimFetch[$1, $3]; }
+      simple_variable                                       { $$ = Expr\Variable[$1]; }
+    | dereferencable '[' optional_expr ']'                  { $$ = Expr\ArrayDimFetch[$1, $3]; }
+    | constant '[' optional_expr ']'                        { $$ = Expr\ArrayDimFetch[$1, $3]; }
+    | dereferencable '{' expr '}'                           { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | function_call                                         { $$ = $1; }
-    | array_object_dereferencable T_OBJECT_OPERATOR property_name argument_list
+    | dereferencable T_OBJECT_OPERATOR property_name argument_list
           { $$ = Expr\MethodCall[$1, $3, $4]; }
-    | array_object_dereferencable T_NULLSAFE_OBJECT_OPERATOR property_name argument_list
-          { $$ = Expr\NullsafeMethodCall[$1, $3, $4]; }
-;
-
-optional_plain_variable:
-      /* empty */                                           { $$ = null; }
-    | plain_variable                                        { $$ = $1; }
 ;
 
 variable:
       callable_variable                                     { $$ = $1; }
     | static_member                                         { $$ = $1; }
-    | array_object_dereferencable T_OBJECT_OPERATOR property_name
-          { $$ = Expr\PropertyFetch[$1, $3]; }
-    | array_object_dereferencable T_NULLSAFE_OBJECT_OPERATOR property_name
-          { $$ = Expr\NullsafePropertyFetch[$1, $3]; }
+    | dereferencable T_OBJECT_OPERATOR property_name        { $$ = Expr\PropertyFetch[$1, $3]; }
 ;
 
 simple_variable:
-      plain_variable                                        { $$ = $1; }
-    | '$' '{' expr '}'                                      { $$ = Expr\Variable[$3]; }
+      T_VARIABLE                                            { $$ = parseVar($1); }
+    | '$' '{' expr '}'                                      { $$ = $3; }
     | '$' simple_variable                                   { $$ = Expr\Variable[$2]; }
-    | '$' error                                             { $$ = Expr\Variable[Expr\Error[]]; $this->errorState = 2; }
+    | '$' error                                             { $$ = Expr\Error[]; $this->errorState = 2; }
 ;
 
 static_member_prop_name:
       simple_variable
-          { $var = $1->name; $$ = \is_string($var) ? Node\VarLikeIdentifier[$var] : $var; }
+          { $var = $1; $$ = \is_string($var) ? Node\VarLikeIdentifier[$var] : $var; }
 ;
 
 static_member:
@@ -1102,11 +920,10 @@ static_member:
 ;
 
 new_variable:
-      simple_variable                                       { $$ = $1; }
+      simple_variable                                       { $$ = Expr\Variable[$1]; }
     | new_variable '[' optional_expr ']'                    { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | new_variable '{' expr '}'                             { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | new_variable T_OBJECT_OPERATOR property_name          { $$ = Expr\PropertyFetch[$1, $3]; }
-    | new_variable T_NULLSAFE_OBJECT_OPERATOR property_name { $$ = Expr\NullsafePropertyFetch[$1, $3]; }
     | class_name T_PAAMAYIM_NEKUDOTAYIM static_member_prop_name
           { $$ = Expr\StaticPropertyFetch[$1, $3]; }
     | new_variable T_PAAMAYIM_NEKUDOTAYIM static_member_prop_name
@@ -1114,20 +931,35 @@ new_variable:
 ;
 
 member_name:
-      identifier_maybe_reserved                             { $$ = $1; }
+      identifier_ex                                         { $$ = $1; }
     | '{' expr '}'	                                        { $$ = $2; }
-    | simple_variable                                       { $$ = $1; }
+    | simple_variable	                                    { $$ = Expr\Variable[$1]; }
 ;
 
 property_name:
-      identifier_not_reserved                               { $$ = $1; }
+      identifier                                            { $$ = $1; }
     | '{' expr '}'	                                        { $$ = $2; }
-    | simple_variable                                       { $$ = $1; }
+    | simple_variable	                                    { $$ = Expr\Variable[$1]; }
     | error                                                 { $$ = Expr\Error[]; $this->errorState = 2; }
 ;
 
 list_expr:
-      T_LIST '(' inner_array_pair_list ')'                  { $$ = Expr\List_[$3]; }
+      T_LIST '(' list_expr_elements ')'                     { $$ = Expr\List_[$3]; }
+;
+
+list_expr_elements:
+      list_expr_elements ',' list_expr_element              { push($1, $3); }
+    | list_expr_element                                     { init($1); }
+;
+
+list_expr_element:
+      variable                                              { $$ = Expr\ArrayItem[$1, null, false]; }
+    | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
+    | list_expr                                             { $$ = Expr\ArrayItem[$1, null, false]; }
+    | expr T_DOUBLE_ARROW variable                          { $$ = Expr\ArrayItem[$3, $1, false]; }
+    | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1, true]; }
+    | expr T_DOUBLE_ARROW list_expr                         { $$ = Expr\ArrayItem[$3, $1, false]; }
+    | /* empty */                                           { $$ = null; }
 ;
 
 array_pair_list:
@@ -1147,12 +979,10 @@ inner_array_pair_list:
 ;
 
 array_pair:
-      expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
-    | ampersand variable                                    { $$ = Expr\ArrayItem[$2, null, true]; }
-    | list_expr                                             { $$ = Expr\ArrayItem[$1, null, false]; }
-    | expr T_DOUBLE_ARROW expr                              { $$ = Expr\ArrayItem[$3, $1,   false]; }
-    | expr T_DOUBLE_ARROW ampersand variable                { $$ = Expr\ArrayItem[$4, $1,   true]; }
-    | expr T_DOUBLE_ARROW list_expr                         { $$ = Expr\ArrayItem[$3, $1,   false]; }
+      expr T_DOUBLE_ARROW expr                              { $$ = Expr\ArrayItem[$3, $1,   false]; }
+    | expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
+    | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
+    | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
     | T_ELLIPSIS expr                                       { $$ = Expr\ArrayItem[$2, null, false, attributes(), true]; }
     | /* empty */                                           { $$ = null; }
 ;
@@ -1175,10 +1005,7 @@ encaps_str_varname:
 encaps_var:
       plain_variable                                        { $$ = $1; }
     | plain_variable '[' encaps_var_offset ']'              { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | plain_variable T_OBJECT_OPERATOR identifier_not_reserved
-          { $$ = Expr\PropertyFetch[$1, $3]; }
-    | plain_variable T_NULLSAFE_OBJECT_OPERATOR identifier_not_reserved
-          { $$ = Expr\NullsafePropertyFetch[$1, $3]; }
+    | plain_variable T_OBJECT_OPERATOR identifier           { $$ = Expr\PropertyFetch[$1, $3]; }
     | T_DOLLAR_OPEN_CURLY_BRACES expr '}'                   { $$ = Expr\Variable[$2]; }
     | T_DOLLAR_OPEN_CURLY_BRACES T_STRING_VARNAME '}'       { $$ = Expr\Variable[$2]; }
     | T_DOLLAR_OPEN_CURLY_BRACES encaps_str_varname '[' expr ']' '}'

grammar/phpyLang.php

@@ -1,192 +0,0 @@
-<?php
-
-///////////////////////////////
-/// Utility regex constants ///
-///////////////////////////////
-
-const LIB = '(?(DEFINE)
-    (?<singleQuotedString>\'[^\\\\\']*+(?:\\\\.[^\\\\\']*+)*+\')
-    (?<doubleQuotedString>"[^\\\\"]*+(?:\\\\.[^\\\\"]*+)*+")
-    (?<string>(?&singleQuotedString)|(?&doubleQuotedString))
-    (?<comment>/\*[^*]*+(?:\*(?!/)[^*]*+)*+\*/)
-    (?<code>\{[^\'"/{}]*+(?:(?:(?&string)|(?&comment)|(?&code)|/)[^\'"/{}]*+)*+})
-)';
-
-const PARAMS = '\[(?<params>[^[\]]*+(?:\[(?&params)\][^[\]]*+)*+)\]';
-const ARGS   = '\((?<args>[^()]*+(?:\((?&args)\)[^()]*+)*+)\)';
-
-///////////////////////////////
-/// Preprocessing functions ///
-///////////////////////////////
-
-function preprocessGrammar($code) {
-    $code = resolveNodes($code);
-    $code = resolveMacros($code);
-    $code = resolveStackAccess($code);
-
-    return $code;
-}
-
-function resolveNodes($code) {
-    return preg_replace_callback(
-        '~\b(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
-        function($matches) {
-            // recurse
-            $matches['params'] = resolveNodes($matches['params']);
-
-            $params = magicSplit(
-                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
-                $matches['params']
-            );
-
-            $paramCode = '';
-            foreach ($params as $param) {
-                $paramCode .= $param . ', ';
-            }
-
-            return 'new ' . $matches['name'] . '(' . $paramCode . 'attributes())';
-        },
-        $code
-    );
-}
-
-function resolveMacros($code) {
-    return preg_replace_callback(
-        '~\b(?<!::|->)(?!array\()(?<name>[a-z][A-Za-z]++)' . ARGS . '~',
-        function($matches) {
-            // recurse
-            $matches['args'] = resolveMacros($matches['args']);
-
-            $name = $matches['name'];
-            $args = magicSplit(
-                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
-                $matches['args']
-            );
-
-            if ('attributes' === $name) {
-                assertArgs(0, $args, $name);
-                return '$this->startAttributeStack[#1] + $this->endAttributes';
-            }
-
-            if ('stackAttributes' === $name) {
-                assertArgs(1, $args, $name);
-                return '$this->startAttributeStack[' . $args[0] . ']'
-                       . ' + $this->endAttributeStack[' . $args[0] . ']';
-            }
-
-            if ('init' === $name) {
-                return '$$ = array(' . implode(', ', $args) . ')';
-            }
-
-            if ('push' === $name) {
-                assertArgs(2, $args, $name);
-
-                return $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0];
-            }
-
-            if ('pushNormalizing' === $name) {
-                assertArgs(2, $args, $name);
-
-                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
-                       . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
-            }
-
-            if ('toArray' == $name) {
-                assertArgs(1, $args, $name);
-
-                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
-            }
-
-            if ('parseVar' === $name) {
-                assertArgs(1, $args, $name);
-
-                return 'substr(' . $args[0] . ', 1)';
-            }
-
-            if ('parseEncapsed' === $name) {
-                assertArgs(3, $args, $name);
-
-                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\Scalar\EncapsedStringPart) {'
-                       . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, ' . $args[1] . ', ' . $args[2] . '); } }';
-            }
-
-            if ('makeNop' === $name) {
-                assertArgs(3, $args, $name);
-
-                return '$startAttributes = ' . $args[1] . ';'
-                       . ' if (isset($startAttributes[\'comments\']))'
-                       . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
-                       . ' else { ' . $args[0] . ' = null; }';
-            }
-
-            if ('makeZeroLengthNop' == $name) {
-                assertArgs(2, $args, $name);
-
-                return '$startAttributes = ' . $args[1] . ';'
-                       . ' if (isset($startAttributes[\'comments\']))'
-                       . ' { ' . $args[0] . ' = new Stmt\Nop($this->createCommentNopAttributes($startAttributes[\'comments\'])); }'
-                       . ' else { ' . $args[0] . ' = null; }';
-            }
-
-            if ('strKind' === $name) {
-                assertArgs(1, $args, $name);
-
-                return '(' . $args[0] . '[0] === "\'" || (' . $args[0] . '[1] === "\'" && '
-                       . '(' . $args[0] . '[0] === \'b\' || ' . $args[0] . '[0] === \'B\')) '
-                       . '? Scalar\String_::KIND_SINGLE_QUOTED : Scalar\String_::KIND_DOUBLE_QUOTED)';
-            }
-
-            if ('prependLeadingComments' === $name) {
-                assertArgs(1, $args, $name);
-
-                return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
-                       . 'if (!empty($attrs[\'comments\'])) {'
-                       . '$stmts[0]->setAttribute(\'comments\', '
-                       . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
-            }
-
-            return $matches[0];
-        },
-        $code
-    );
-}
-
-function assertArgs($num, $args, $name) {
-    if ($num != count($args)) {
-        die('Wrong argument count for ' . $name . '().');
-    }
-}
-
-function resolveStackAccess($code) {
-    $code = preg_replace('/\$\d+/', '$this->semStack[$0]', $code);
-    $code = preg_replace('/#(\d+)/', '$$1', $code);
-    return $code;
-}
-
-function removeTrailingWhitespace($code) {
-    $lines = explode("\n", $code);
-    $lines = array_map('rtrim', $lines);
-    return implode("\n", $lines);
-}
-
-//////////////////////////////
-/// Regex helper functions ///
-//////////////////////////////
-
-function regex($regex) {
-    return '~' . LIB . '(?:' . str_replace('~', '\~', $regex) . ')~';
-}
-
-function magicSplit($regex, $string) {
-    $pieces = preg_split(regex('(?:(?&string)|(?&comment)|(?&code))(*SKIP)(*FAIL)|' . $regex), $string);
-
-    foreach ($pieces as &$piece) {
-        $piece = trim($piece);
-    }
-
-    if ($pieces === ['']) {
-        return [];
-    }
-
-    return $pieces;
-}

grammar/rebuildParsers.php

@@ -1,7 +1,5 @@
 <?php
 
-require __DIR__ . '/phpyLang.php';
-
 $grammarFileToName = [
     __DIR__ . '/php5.y' => 'Php5',
     __DIR__ . '/php7.y' => 'Php7',
@@ -15,16 +13,34 @@ $tmpResultFile  = __DIR__ . '/tmp_parser.php';
 $resultDir = __DIR__ . '/../lib/PhpParser/Parser';
 $tokensResultsFile = $resultDir . '/Tokens.php';
 
-$kmyacc = getenv('KMYACC');
-if (!$kmyacc) {
-    // Use phpyacc from dev dependencies by default.
-    $kmyacc = __DIR__ . '/../vendor/bin/phpyacc';
+// check for kmyacc binary in this directory, otherwise fall back to global name
+if (file_exists(__DIR__ . '/kmyacc.exe')) {
+    $kmyacc = __DIR__ . '/kmyacc.exe';
+} else if (file_exists(__DIR__ . '/kmyacc')) {
+    $kmyacc = __DIR__ . '/kmyacc';
+} else {
+    $kmyacc = 'kmyacc';
 }
 
 $options = array_flip($argv);
 $optionDebug = isset($options['--debug']);
 $optionKeepTmpGrammar = isset($options['--keep-tmp-grammar']);
 
+///////////////////////////////
+/// Utility regex constants ///
+///////////////////////////////
+
+const LIB = '(?(DEFINE)
+    (?<singleQuotedString>\'[^\\\\\']*+(?:\\\\.[^\\\\\']*+)*+\')
+    (?<doubleQuotedString>"[^\\\\"]*+(?:\\\\.[^\\\\"]*+)*+")
+    (?<string>(?&singleQuotedString)|(?&doubleQuotedString))
+    (?<comment>/\*[^*]*+(?:\*(?!/)[^*]*+)*+\*/)
+    (?<code>\{[^\'"/{}]*+(?:(?:(?&string)|(?&comment)|(?&code)|/)[^\'"/{}]*+)*+})
+)';
+
+const PARAMS = '\[(?<params>[^[\]]*+(?:\[(?&params)\][^[\]]*+)*+)\]';
+const ARGS   = '\((?<args>[^()]*+(?:\((?&args)\)[^()]*+)*+)\)';
+
 ///////////////////
 /// Main script ///
 ///////////////////
@@ -36,14 +52,18 @@ foreach ($grammarFileToName as $grammarFile => $name) {
 
     $grammarCode = file_get_contents($grammarFile);
     $grammarCode = str_replace('%tokens', $tokens, $grammarCode);
-    $grammarCode = preprocessGrammar($grammarCode);
+
+    $grammarCode = resolveNodes($grammarCode);
+    $grammarCode = resolveMacros($grammarCode);
+    $grammarCode = resolveStackAccess($grammarCode);
 
     file_put_contents($tmpGrammarFile, $grammarCode);
 
     $additionalArgs = $optionDebug ? '-t -v' : '';
 
     echo "Building $name parser.\n";
-    $output = execCmd("$kmyacc $additionalArgs -m $skeletonFile -p $name $tmpGrammarFile");
+    $output = trim(shell_exec("$kmyacc $additionalArgs -l -m $skeletonFile -p $name $tmpGrammarFile 2>&1"));
+    echo "Output: \"$output\"\n";
 
     $resultCode = file_get_contents($tmpResultFile);
     $resultCode = removeTrailingWhitespace($resultCode);
@@ -53,7 +73,8 @@ foreach ($grammarFileToName as $grammarFile => $name) {
     unlink($tmpResultFile);
 
     echo "Building token definition.\n";
-    $output = execCmd("$kmyacc -m $tokensTemplate $tmpGrammarFile");
+    $output = trim(shell_exec("$kmyacc -l -m $tokensTemplate $tmpGrammarFile 2>&1"));
+    assert($output === '');
     rename($tmpResultFile, $tokensResultsFile);
 
     if (!$optionKeepTmpGrammar) {
@@ -61,9 +82,151 @@ foreach ($grammarFileToName as $grammarFile => $name) {
     }
 }
 
-////////////////////////////////
-/// Utility helper functions ///
-////////////////////////////////
+///////////////////////////////
+/// Preprocessing functions ///
+///////////////////////////////
+
+function resolveNodes($code) {
+    return preg_replace_callback(
+        '~\b(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
+        function($matches) {
+            // recurse
+            $matches['params'] = resolveNodes($matches['params']);
+
+            $params = magicSplit(
+                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
+                $matches['params']
+            );
+
+            $paramCode = '';
+            foreach ($params as $param) {
+                $paramCode .= $param . ', ';
+            }
+
+            return 'new ' . $matches['name'] . '(' . $paramCode . 'attributes())';
+        },
+        $code
+    );
+}
+
+function resolveMacros($code) {
+    return preg_replace_callback(
+        '~\b(?<!::|->)(?!array\()(?<name>[a-z][A-Za-z]++)' . ARGS . '~',
+        function($matches) {
+            // recurse
+            $matches['args'] = resolveMacros($matches['args']);
+
+            $name = $matches['name'];
+            $args = magicSplit(
+                '(?:' . PARAMS . '|' . ARGS . ')(*SKIP)(*FAIL)|,',
+                $matches['args']
+            );
+
+            if ('attributes' == $name) {
+                assertArgs(0, $args, $name);
+                return '$this->startAttributeStack[#1] + $this->endAttributes';
+            }
+
+            if ('stackAttributes' == $name) {
+                assertArgs(1, $args, $name);
+                return '$this->startAttributeStack[' . $args[0] . ']'
+                     . ' + $this->endAttributeStack[' . $args[0] . ']';
+            }
+
+            if ('init' == $name) {
+                return '$$ = array(' . implode(', ', $args) . ')';
+            }
+
+            if ('push' == $name) {
+                assertArgs(2, $args, $name);
+
+                return $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0];
+            }
+
+            if ('pushNormalizing' == $name) {
+                assertArgs(2, $args, $name);
+
+                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
+                     . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
+            }
+
+            if ('toArray' == $name) {
+                assertArgs(1, $args, $name);
+
+                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
+            }
+
+            if ('parseVar' == $name) {
+                assertArgs(1, $args, $name);
+
+                return 'substr(' . $args[0] . ', 1)';
+            }
+
+            if ('parseEncapsed' == $name) {
+                assertArgs(3, $args, $name);
+
+                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\Scalar\EncapsedStringPart) {'
+                     . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, ' . $args[1] . ', ' . $args[2] . '); } }';
+            }
+
+            if ('makeNop' == $name) {
+                assertArgs(3, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                . ' if (isset($startAttributes[\'comments\']))'
+                . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
+                . ' else { ' . $args[0] . ' = null; }';
+            }
+
+            if ('makeZeroLengthNop' == $name) {
+                assertArgs(2, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                    . ' if (isset($startAttributes[\'comments\']))'
+                    . ' { ' . $args[0] . ' = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); }'
+                    . ' else { ' . $args[0] . ' = null; }';
+            }
+
+            if ('strKind' == $name) {
+                assertArgs(1, $args, $name);
+
+                return '(' . $args[0] . '[0] === "\'" || (' . $args[0] . '[1] === "\'" && '
+                     . '(' . $args[0] . '[0] === \'b\' || ' . $args[0] . '[0] === \'B\')) '
+                     . '? Scalar\String_::KIND_SINGLE_QUOTED : Scalar\String_::KIND_DOUBLE_QUOTED)';
+            }
+
+            if ('prependLeadingComments' == $name) {
+                assertArgs(1, $args, $name);
+
+                return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
+                . 'if (!empty($attrs[\'comments\'])) {'
+                . '$stmts[0]->setAttribute(\'comments\', '
+                . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
+            }
+
+            return $matches[0];
+        },
+        $code
+    );
+}
+
+function assertArgs($num, $args, $name) {
+    if ($num != count($args)) {
+        die('Wrong argument count for ' . $name . '().');
+    }
+}
+
+function resolveStackAccess($code) {
+    $code = preg_replace('/\$\d+/', '$this->semStack[$0]', $code);
+    $code = preg_replace('/#(\d+)/', '$$1', $code);
+    return $code;
+}
+
+function removeTrailingWhitespace($code) {
+    $lines = explode("\n", $code);
+    $lines = array_map('rtrim', $lines);
+    return implode("\n", $lines);
+}
 
 function ensureDirExists($dir) {
     if (!is_dir($dir)) {
@@ -71,11 +234,24 @@ function ensureDirExists($dir) {
     }
 }
 
-function execCmd($cmd) {
-    $output = trim(shell_exec("$cmd 2>&1"));
-    if ($output !== "") {
-        echo "> " . $cmd . "\n";
-        echo $output;
+//////////////////////////////
+/// Regex helper functions ///
+//////////////////////////////
+
+function regex($regex) {
+    return '~' . LIB . '(?:' . str_replace('~', '\~', $regex) . ')~';
 }
-    return $output;
+
+function magicSplit($regex, $string) {
+    $pieces = preg_split(regex('(?:(?&string)|(?&comment)|(?&code))(*SKIP)(*FAIL)|' . $regex), $string);
+
+    foreach ($pieces as &$piece) {
+        $piece = trim($piece);
+    }
+
+    if ($pieces === ['']) {
+        return [];
+    }
+
+    return $pieces;
 }

grammar/tokens.y

@@ -1,7 +1,6 @@
 /* We currently rely on the token ID mapping to be the same between PHP 5 and PHP 7 - so the same lexer can be used for
  * both. This is enforced by sharing this token file. */
 
-%right T_THROW
 %left T_INCLUDE T_INCLUDE_ONCE T_EVAL T_REQUIRE T_REQUIRE_ONCE
 %left ','
 %left T_LOGICAL_OR
@@ -18,7 +17,7 @@
 %left T_BOOLEAN_AND
 %left '|'
 %left '^'
-%left T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
+%left '&'
 %nonassoc T_IS_EQUAL T_IS_NOT_EQUAL T_IS_IDENTICAL T_IS_NOT_IDENTICAL T_SPACESHIP
 %nonassoc '<' T_IS_SMALLER_OR_EQUAL '>' T_IS_GREATER_OR_EQUAL
 %left T_SL T_SR
@@ -42,6 +41,8 @@
 %token T_VARIABLE
 %token T_NUM_STRING
 %token T_INLINE_HTML
+%token T_CHARACTER
+%token T_BAD_CHARACTER
 %token T_ENCAPSED_AND_WHITESPACE
 %token T_CONSTANT_ENCAPSED_STRING
 %token T_ECHO
@@ -56,7 +57,6 @@
 %token T_ENDDECLARE
 %token T_AS
 %token T_SWITCH
-%token T_MATCH
 %token T_ENDSWITCH
 %token T_CASE
 %token T_DEFAULT
@@ -74,7 +74,7 @@
 %token T_USE
 %token T_INSTEADOF
 %token T_GLOBAL
-%right T_STATIC T_ABSTRACT T_FINAL T_PRIVATE T_PROTECTED T_PUBLIC T_READONLY
+%right T_STATIC T_ABSTRACT T_FINAL T_PRIVATE T_PROTECTED T_PUBLIC
 %token T_VAR
 %token T_UNSET
 %token T_ISSET
@@ -83,11 +83,9 @@
 %token T_CLASS
 %token T_TRAIT
 %token T_INTERFACE
-%token T_ENUM
 %token T_EXTENDS
 %token T_IMPLEMENTS
 %token T_OBJECT_OPERATOR
-%token T_NULLSAFE_OBJECT_OPERATOR
 %token T_DOUBLE_ARROW
 %token T_LIST
 %token T_ARRAY
@@ -98,6 +96,12 @@
 %token T_FUNC_C
 %token T_LINE
 %token T_FILE
+%token T_COMMENT
+%token T_DOC_COMMENT
+%token T_OPEN_TAG
+%token T_OPEN_TAG_WITH_ECHO
+%token T_CLOSE_TAG
+%token T_WHITESPACE
 %token T_START_HEREDOC
 %token T_END_HEREDOC
 %token T_DOLLAR_OPEN_CURLY_BRACES
@@ -108,8 +112,3 @@
 %token T_DIR
 %token T_NS_SEPARATOR
 %token T_ELLIPSIS
-%token T_NAME_FULLY_QUALIFIED
-%token T_NAME_QUALIFIED
-%token T_NAME_RELATIVE
-%token T_ATTRIBUTE
-%token T_ENUM

lib/PhpParser/Builder/ClassConst.php

@@ -1,132 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace PhpParser\Builder;
-
-use PhpParser;
-use PhpParser\BuilderHelpers;
-use PhpParser\Node;
-use PhpParser\Node\Const_;
-use PhpParser\Node\Identifier;
-use PhpParser\Node\Stmt;
-
-class ClassConst implements PhpParser\Builder
-{
-    protected $flags = 0;
-    protected $attributes = [];
-    protected $constants = [];
-
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
-    /**
-     * Creates a class constant builder
-     *
-     * @param string|Identifier                          $name  Name
-     * @param Node\Expr|bool|null|int|float|string|array $value Value
-     */
-    public function __construct($name, $value) {
-        $this->constants = [new Const_($name, BuilderHelpers::normalizeValue($value))];
-    }
-
-    /**
-     * Add another constant to const group
-     *
-     * @param string|Identifier                          $name  Name
-     * @param Node\Expr|bool|null|int|float|string|array $value Value
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addConst($name, $value) {
-        $this->constants[] = new Const_($name, BuilderHelpers::normalizeValue($value));
-
-        return $this;
-    }
-
-    /**
-     * Makes the constant public.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makePublic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PUBLIC);
-
-        return $this;
-    }
-
-    /**
-     * Makes the constant protected.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makeProtected() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PROTECTED);
-
-        return $this;
-    }
-
-    /**
-     * Makes the constant private.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makePrivate() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PRIVATE);
-
-        return $this;
-    }
-
-    /**
-     * Makes the constant final.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makeFinal() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
-
-        return $this;
-    }
-
-    /**
-     * Sets doc comment for the constant.
-     *
-     * @param PhpParser\Comment\Doc|string $docComment Doc comment to set
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function setDocComment($docComment) {
-        $this->attributes = [
-            'comments' => [BuilderHelpers::normalizeDocComment($docComment)]
-        ];
-
-        return $this;
-    }
-
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
-    /**
-     * Returns the built class node.
-     *
-     * @return Stmt\ClassConst The built constant node
-     */
-    public function getNode(): PhpParser\Node {
-        return new Stmt\ClassConst(
-            $this->constants,
-            $this->flags,
-            $this->attributes,
-            $this->attributeGroups
-        );
-    }
-}

lib/PhpParser/Builder/Class_.php

@@ -4,7 +4,6 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Node;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
@@ -21,9 +20,6 @@ class Class_ extends Declaration
     protected $properties = [];
     protected $methods = [];
 
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
     /**
      * Creates a class builder.
      *
@@ -110,19 +106,6 @@ class Class_ extends Declaration
         return $this;
     }
 
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
     /**
      * Returns the built class node.
      *
@@ -134,7 +117,6 @@ class Class_ extends Declaration
             'extends' => $this->extends,
             'implements' => $this->implements,
             'stmts' => array_merge($this->uses, $this->constants, $this->properties, $this->methods),
-            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/EnumCase.php

@@ -1,85 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace PhpParser\Builder;
-
-use PhpParser;
-use PhpParser\BuilderHelpers;
-use PhpParser\Node;
-use PhpParser\Node\Identifier;
-use PhpParser\Node\Stmt;
-
-class EnumCase implements PhpParser\Builder
-{
-    protected $name;
-    protected $value = null;
-    protected $attributes = [];
-
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
-    /**
-     * Creates an enum case builder.
-     *
-     * @param string|Identifier $name  Name
-     */
-    public function __construct($name) {
-        $this->name = $name;
-    }
-
-    /**
-     * Sets the value.
-     *
-     * @param Node\Expr|string|int $value
-     *
-     * @return $this
-     */
-    public function setValue($value) {
-        $this->value = BuilderHelpers::normalizeValue($value);
-
-        return $this;
-    }
-
-    /**
-     * Sets doc comment for the constant.
-     *
-     * @param PhpParser\Comment\Doc|string $docComment Doc comment to set
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function setDocComment($docComment) {
-        $this->attributes = [
-            'comments' => [BuilderHelpers::normalizeDocComment($docComment)]
-        ];
-
-        return $this;
-    }
-
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
-    /**
-     * Returns the built enum case node.
-     *
-     * @return Stmt\EnumCase The built constant node
-     */
-    public function getNode(): PhpParser\Node {
-        return new Stmt\EnumCase(
-            $this->name,
-            $this->value,
-            $this->attributes,
-            $this->attributeGroups
-        );
-    }
-}

lib/PhpParser/Builder/Enum_.php

@@ -1,117 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Builder;
-
-use PhpParser;
-use PhpParser\BuilderHelpers;
-use PhpParser\Node;
-use PhpParser\Node\Identifier;
-use PhpParser\Node\Name;
-use PhpParser\Node\Stmt;
-
-class Enum_ extends Declaration
-{
-    protected $name;
-    protected $scalarType = null;
-
-    protected $implements = [];
-
-    protected $uses = [];
-    protected $enumCases = [];
-    protected $constants = [];
-    protected $methods = [];
-
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
-    /**
-     * Creates an enum builder.
-     *
-     * @param string $name Name of the enum
-     */
-    public function __construct(string $name) {
-        $this->name = $name;
-    }
-
-    /**
-     * Sets the scalar type.
-     *
-     * @param string|Identifier $type
-     *
-     * @return $this
-     */
-    public function setScalarType($scalarType) {
-        $this->scalarType = BuilderHelpers::normalizeType($scalarType);
-
-        return $this;
-    }
-
-    /**
-     * Implements one or more interfaces.
-     *
-     * @param Name|string ...$interfaces Names of interfaces to implement
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function implement(...$interfaces) {
-        foreach ($interfaces as $interface) {
-            $this->implements[] = BuilderHelpers::normalizeName($interface);
-        }
-
-        return $this;
-    }
-
-    /**
-     * Adds a statement.
-     *
-     * @param Stmt|PhpParser\Builder $stmt The statement to add
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addStmt($stmt) {
-        $stmt = BuilderHelpers::normalizeNode($stmt);
-
-        $targets = [
-            Stmt\TraitUse::class    => &$this->uses,
-            Stmt\EnumCase::class    => &$this->enumCases,
-            Stmt\ClassConst::class  => &$this->constants,
-            Stmt\ClassMethod::class => &$this->methods,
-        ];
-
-        $class = \get_class($stmt);
-        if (!isset($targets[$class])) {
-            throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
-        }
-
-        $targets[$class][] = $stmt;
-
-        return $this;
-    }
-
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
-    /**
-     * Returns the built class node.
-     *
-     * @return Stmt\Enum_ The built enum node
-     */
-    public function getNode() : PhpParser\Node {
-        return new Stmt\Enum_($this->name, [
-            'scalarType' => $this->scalarType,
-            'implements' => $this->implements,
-            'stmts' => array_merge($this->uses, $this->enumCases, $this->constants, $this->methods),
-            'attrGroups' => $this->attributeGroups,
-        ], $this->attributes);
-    }
-}

lib/PhpParser/Builder/FunctionLike.php

@@ -61,7 +61,8 @@ abstract class FunctionLike extends Declaration
     /**
      * Sets the return type for PHP 7.
      *
-     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type
+     * @param string|Node\Name|Node\NullableType $type One of array, callable, string, int, float,
+     *                                                 bool, iterable, or a class/interface name.
      *
      * @return $this The builder instance (for fluid interface)
      */

lib/PhpParser/Builder/Function_.php

@@ -12,9 +12,6 @@ class Function_ extends FunctionLike
     protected $name;
     protected $stmts = [];
 
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
     /**
      * Creates a function builder.
      *
@@ -37,19 +34,6 @@ class Function_ extends FunctionLike
         return $this;
     }
 
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
     /**
      * Returns the built function node.
      *
@@ -61,7 +45,6 @@ class Function_ extends FunctionLike
             'params'     => $this->params,
             'returnType' => $this->returnType,
             'stmts'      => $this->stmts,
-            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Interface_.php

@@ -4,7 +4,6 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Node;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
@@ -15,9 +14,6 @@ class Interface_ extends Declaration
     protected $constants = [];
     protected $methods = [];
 
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
     /**
      * Creates an interface builder.
      *
@@ -65,19 +61,6 @@ class Interface_ extends Declaration
         return $this;
     }
 
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
     /**
      * Returns the built interface node.
      *
@@ -87,7 +70,6 @@ class Interface_ extends Declaration
         return new Stmt\Interface_($this->name, [
             'extends' => $this->extends,
             'stmts' => array_merge($this->constants, $this->methods),
-            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Method.php

@@ -15,9 +15,6 @@ class Method extends FunctionLike
     /** @var array|null */
     protected $stmts = [];
 
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
     /**
      * Creates a method builder.
      *
@@ -115,19 +112,6 @@ class Method extends FunctionLike
         return $this;
     }
 
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
     /**
      * Returns the built method node.
      *
@@ -140,7 +124,6 @@ class Method extends FunctionLike
             'params'     => $this->params,
             'returnType' => $this->returnType,
             'stmts'      => $this->stmts,
-            'attrGroups' => $this->attributeGroups,
         ], $this->attributes);
     }
 }

lib/PhpParser/Builder/Namespace_.php

@@ -37,7 +37,7 @@ class Namespace_ extends Declaration
     /**
      * Returns the built node.
      *
-     * @return Stmt\Namespace_ The built node
+     * @return Node The built node
      */
     public function getNode() : Node {
         return new Stmt\Namespace_($this->name, $this->stmts, $this->attributes);

lib/PhpParser/Builder/Param.php

@@ -12,16 +12,13 @@ class Param implements PhpParser\Builder
 
     protected $default = null;
 
-    /** @var Node\Identifier|Node\Name|Node\NullableType|null */
+    /** @var string|Node\Name|Node\NullableType|null */
     protected $type = null;
 
     protected $byRef = false;
 
     protected $variadic = false;
 
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
     /**
      * Creates a parameter builder.
      *
@@ -47,7 +44,7 @@ class Param implements PhpParser\Builder
     /**
      * Sets type for the parameter.
      *
-     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type Parameter type
+     * @param string|Node\Name|Node\NullableType $type Parameter type
      *
      * @return $this The builder instance (for fluid interface)
      */
@@ -63,7 +60,7 @@ class Param implements PhpParser\Builder
     /**
      * Sets type for the parameter.
      *
-     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type Parameter type
+     * @param string|Node\Name|Node\NullableType $type Parameter type
      *
      * @return $this The builder instance (for fluid interface)
      *
@@ -95,19 +92,6 @@ class Param implements PhpParser\Builder
         return $this;
     }
 
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
     /**
      * Returns the built parameter node.
      *
@@ -116,7 +100,7 @@ class Param implements PhpParser\Builder
     public function getNode() : Node {
         return new Node\Param(
             new Node\Expr\Variable($this->name),
-            $this->default, $this->type, $this->byRef, $this->variadic, [], 0, $this->attributeGroups
+            $this->default, $this->type, $this->byRef, $this->variadic
         );
     }
 }

lib/PhpParser/Builder/Property.php

@@ -4,11 +4,10 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Node;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
+use PhpParser\Node\NullableType;
 use PhpParser\Node\Stmt;
-use PhpParser\Node\ComplexType;
 
 class Property implements PhpParser\Builder
 {
@@ -21,9 +20,6 @@ class Property implements PhpParser\Builder
     /** @var null|Identifier|Name|NullableType */
     protected $type;
 
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
     /**
      * Creates a property builder.
      *
@@ -77,17 +73,6 @@ class Property implements PhpParser\Builder
         return $this;
     }
 
-    /**
-     * Makes the property readonly.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makeReadonly() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_READONLY);
-
-        return $this;
-    }
-
     /**
      * Sets default value for the property.
      *
@@ -119,7 +104,7 @@ class Property implements PhpParser\Builder
     /**
      * Sets the property type for PHP 7.4+.
      *
-     * @param string|Name|Identifier|ComplexType $type
+     * @param string|Name|NullableType|Identifier $type
      *
      * @return $this
      */
@@ -129,19 +114,6 @@ class Property implements PhpParser\Builder
         return $this;
     }
 
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
     /**
      * Returns the built class node.
      *
@@ -154,8 +126,7 @@ class Property implements PhpParser\Builder
                 new Stmt\PropertyProperty($this->name, $this->default)
             ],
             $this->attributes,
-            $this->type,
-            $this->attributeGroups
+            $this->type
         );
     }
 }

lib/PhpParser/Builder/Trait_.php

@@ -4,7 +4,6 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
 class Trait_ extends Declaration
@@ -14,9 +13,6 @@ class Trait_ extends Declaration
     protected $properties = [];
     protected $methods = [];
 
-    /** @var Node\AttributeGroup[] */
-    protected $attributeGroups = [];
-
     /**
      * Creates an interface builder.
      *
@@ -49,19 +45,6 @@ class Trait_ extends Declaration
         return $this;
     }
 
-    /**
-     * Adds an attribute group.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addAttribute($attribute) {
-        $this->attributeGroups[] = BuilderHelpers::normalizeAttribute($attribute);
-
-        return $this;
-    }
-
     /**
      * Returns the built trait node.
      *
@@ -70,8 +53,7 @@ class Trait_ extends Declaration
     public function getNode() : PhpParser\Node {
         return new Stmt\Trait_(
             $this->name, [
-                'stmts' => array_merge($this->uses, $this->properties, $this->methods),
-                'attrGroups' => $this->attributeGroups,
+                'stmts' => array_merge($this->uses, $this->properties, $this->methods)
             ], $this->attributes
         );
     }

lib/PhpParser/Builder/Use_.php

@@ -39,7 +39,7 @@ class Use_ implements Builder
     /**
      * Returns the built node.
      *
-     * @return Stmt\Use_ The built node
+     * @return Node The built node
      */
     public function getNode() : Node {
         return new Stmt\Use_([

lib/PhpParser/BuilderFactory.php

@@ -12,21 +12,6 @@ use PhpParser\Node\Stmt\Use_;
 
 class BuilderFactory
 {
-    /**
-     * Creates an attribute node.
-     *
-     * @param string|Name $name Name of the attribute
-     * @param array       $args Attribute named arguments
-     *
-     * @return Node\Attribute
-     */
-    public function attribute($name, array $args = []) : Node\Attribute {
-        return new Node\Attribute(
-            BuilderHelpers::normalizeName($name),
-            $this->args($args)
-        );
-    }
-
     /**
      * Creates a namespace builder.
      *
@@ -71,17 +56,6 @@ class BuilderFactory
         return new Builder\Trait_($name);
     }
 
-    /**
-     * Creates an enum builder.
-     *
-     * @param string $name Name of the enum
-     *
-     * @return Builder\Enum_ The created enum builder
-     */
-    public function enum(string $name) : Builder\Enum_ {
-        return new Builder\Enum_($name);
-    }
-
     /**
      * Creates a trait use builder.
      *
@@ -187,29 +161,6 @@ class BuilderFactory
         return new Builder\Use_($name, Use_::TYPE_CONSTANT);
     }
 
-    /**
-     * Creates a class constant builder.
-     *
-     * @param string|Identifier                          $name  Name
-     * @param Node\Expr|bool|null|int|float|string|array $value Value
-     *
-     * @return Builder\ClassConst The created use const builder
-     */
-    public function classConst($name, $value) : Builder\ClassConst {
-        return new Builder\ClassConst($name, $value);
-    }
-
-    /**
-     * Creates an enum case builder.
-     *
-     * @param string|Identifier $name  Name
-     *
-     * @return Builder\EnumCase The created use const builder
-     */
-    public function enumCase($name) : Builder\EnumCase {
-        return new Builder\EnumCase($name);
-    }
-
     /**
      * Creates node a for a literal value.
      *
@@ -247,14 +198,12 @@ class BuilderFactory
      */
     public function args(array $args) : array {
         $normalizedArgs = [];
-        foreach ($args as $key => $arg) {
-            if (!($arg instanceof Arg)) {
-                $arg = new Arg(BuilderHelpers::normalizeValue($arg));
-            }
-            if (\is_string($key)) {
-                $arg->name = BuilderHelpers::normalizeIdentifier($key);
-            }
+        foreach ($args as $arg) {
+            if ($arg instanceof Arg) {
                 $normalizedArgs[] = $arg;
+            } else {
+                $normalizedArgs[] = new Arg(BuilderHelpers::normalizeValue($arg));
+            }
         }
         return $normalizedArgs;
     }

lib/PhpParser/BuilderHelpers.php

@@ -2,7 +2,6 @@
 
 namespace PhpParser;
 
-use PhpParser\Node\ComplexType;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
@@ -27,9 +26,7 @@ final class BuilderHelpers
     public static function normalizeNode($node) : Node {
         if ($node instanceof Builder) {
             return $node->getNode();
-        }
-
-        if ($node instanceof Node) {
+        } elseif ($node instanceof Node) {
             return $node;
         }
 
@@ -104,27 +101,7 @@ final class BuilderHelpers
      * @return Name The normalized name
      */
     public static function normalizeName($name) : Name {
-        if ($name instanceof Name) {
-            return $name;
-        }
-
-        if (is_string($name)) {
-            if (!$name) {
-                throw new \LogicException('Name cannot be empty');
-            }
-
-            if ($name[0] === '\\') {
-                return new Name\FullyQualified(substr($name, 1));
-            }
-
-            if (0 === strpos($name, 'namespace\\')) {
-                return new Name\Relative(substr($name, strlen('namespace\\')));
-            }
-
-            return new Name($name);
-        }
-
-        throw new \LogicException('Name must be a string or an instance of Node\Name');
+        return self::normalizeNameCommon($name, false);
     }
 
     /**
@@ -135,17 +112,44 @@ final class BuilderHelpers
      * @return Name|Expr The normalized name or expression
      */
     public static function normalizeNameOrExpr($name) {
+        return self::normalizeNameCommon($name, true);
+    }
+
+    /**
+     * Normalizes a name: Converts string names to Name nodes, optionally allowing expressions.
+     *
+     * @param Expr|Name|string $name      The name to normalize
+     * @param bool             $allowExpr Whether to also allow expressions
+     *
+     * @return Name|Expr The normalized name, or expression (if allowed)
+     */
+    private static function normalizeNameCommon($name, bool $allowExpr) {
+        if ($name instanceof Name) {
+            return $name;
+        } elseif (is_string($name)) {
+            if (!$name) {
+                throw new \LogicException('Name cannot be empty');
+            }
+
+            if ($name[0] === '\\') {
+                return new Name\FullyQualified(substr($name, 1));
+            } elseif (0 === strpos($name, 'namespace\\')) {
+                return new Name\Relative(substr($name, strlen('namespace\\')));
+            } else {
+                return new Name($name);
+            }
+        }
+
+        if ($allowExpr) {
             if ($name instanceof Expr) {
                 return $name;
             }
-
-        if (!is_string($name) && !($name instanceof Name)) {
             throw new \LogicException(
                 'Name must be a string or an instance of Node\Name or Node\Expr'
             );
+        } else {
+            throw new \LogicException('Name must be a string or an instance of Node\Name');
         }
-
-        return self::normalizeName($name);
     }
 
     /**
@@ -154,19 +158,16 @@ final class BuilderHelpers
      * In particular, builtin types become Identifiers, custom types become Names and nullables
      * are wrapped in NullableType nodes.
      *
-     * @param string|Name|Identifier|ComplexType $type The type to normalize
+     * @param string|Name|Identifier|NullableType $type The type to normalize
      *
-     * @return Name|Identifier|ComplexType The normalized type
+     * @return Name|Identifier|NullableType The normalized type
      */
     public static function normalizeType($type) {
         if (!is_string($type)) {
-            if (
-                !$type instanceof Name && !$type instanceof Identifier &&
-                !$type instanceof ComplexType
-            ) {
+            if (!$type instanceof Name && !$type instanceof Identifier
+                    && !$type instanceof NullableType) {
                 throw new \LogicException(
-                    'Type must be a string, or an instance of Name, Identifier or ComplexType'
-                );
+                    'Type must be a string, or an instance of Name, Identifier or NullableType');
             }
             return $type;
         }
@@ -178,7 +179,7 @@ final class BuilderHelpers
         }
 
         $builtinTypes = [
-            'array', 'callable', 'string', 'int', 'float', 'bool', 'iterable', 'void', 'object', 'mixed', 'never',
+            'array', 'callable', 'string', 'int', 'float', 'bool', 'iterable', 'void', 'object'
         ];
 
         $lowerType = strtolower($type);
@@ -188,14 +189,11 @@ final class BuilderHelpers
             $type = self::normalizeName($type);
         }
 
-        $notNullableTypes = [
-            'void', 'mixed', 'never',
-        ];
-        if ($nullable && in_array((string) $type, $notNullableTypes)) {
-            throw new \LogicException(sprintf('%s type cannot be nullable', $type));
+        if ($nullable && (string) $type === 'void') {
+            throw new \LogicException('void type cannot be nullable');
         }
 
-        return $nullable ? new NullableType($type) : $type;
+        return $nullable ? new Node\NullableType($type) : $type;
     }
 
     /**
@@ -209,33 +207,21 @@ final class BuilderHelpers
     public static function normalizeValue($value) : Expr {
         if ($value instanceof Node\Expr) {
             return $value;
-        }
-
-        if (is_null($value)) {
+        } elseif (is_null($value)) {
             return new Expr\ConstFetch(
                 new Name('null')
             );
-        }
-
-        if (is_bool($value)) {
+        } elseif (is_bool($value)) {
             return new Expr\ConstFetch(
                 new Name($value ? 'true' : 'false')
             );
-        }
-
-        if (is_int($value)) {
+        } elseif (is_int($value)) {
             return new Scalar\LNumber($value);
-        }
-
-        if (is_float($value)) {
+        } elseif (is_float($value)) {
             return new Scalar\DNumber($value);
-        }
-
-        if (is_string($value)) {
+        } elseif (is_string($value)) {
             return new Scalar\String_($value);
-        }
-
-        if (is_array($value)) {
+        } elseif (is_array($value)) {
             $items = [];
             $lastKey = -1;
             foreach ($value as $itemKey => $itemValue) {
@@ -254,10 +240,10 @@ final class BuilderHelpers
             }
 
             return new Expr\Array_($items);
-        }
-
+        } else {
             throw new \LogicException('Invalid value');
         }
+    }
 
     /**
      * Normalizes a doc comment: Converts plain strings to PhpParser\Comment\Doc.
@@ -269,33 +255,11 @@ final class BuilderHelpers
     public static function normalizeDocComment($docComment) : Comment\Doc {
         if ($docComment instanceof Comment\Doc) {
             return $docComment;
-        }
-
-        if (is_string($docComment)) {
+        } elseif (is_string($docComment)) {
             return new Comment\Doc($docComment);
-        }
-
+        } else {
             throw new \LogicException('Doc comment must be a string or an instance of PhpParser\Comment\Doc');
         }
-
-    /**
-     * Normalizes a attribute: Converts attribute to the Attribute Group if needed.
-     *
-     * @param Node\Attribute|Node\AttributeGroup $attribute
-     *
-     * @return Node\AttributeGroup The Attribute Group
-     */
-    public static function normalizeAttribute($attribute) : Node\AttributeGroup
-    {
-        if ($attribute instanceof Node\AttributeGroup) {
-            return $attribute;
-        }
-
-        if (!($attribute instanceof Node\Attribute)) {
-            throw new \LogicException('Attribute must be an instance of PhpParser\Node\Attribute or PhpParser\Node\AttributeGroup');
-        }
-
-        return new Node\AttributeGroup([$attribute]);
     }
 
     /**

lib/PhpParser/Comment.php

@@ -5,12 +5,9 @@ namespace PhpParser;
 class Comment implements \JsonSerializable
 {
     protected $text;
-    protected $startLine;
-    protected $startFilePos;
-    protected $startTokenPos;
-    protected $endLine;
-    protected $endFilePos;
-    protected $endTokenPos;
+    protected $line;
+    protected $filePos;
+    protected $tokenPos;
 
     /**
      * Constructs a comment node.
@@ -21,17 +18,12 @@ class Comment implements \JsonSerializable
      * @param int    $startTokenPos Token offset the comment started on
      */
     public function __construct(
-        string $text,
-        int $startLine = -1, int $startFilePos = -1, int $startTokenPos = -1,
-        int $endLine = -1, int $endFilePos = -1, int $endTokenPos = -1
+        string $text, int $startLine = -1, int $startFilePos = -1, int $startTokenPos = -1
     ) {
         $this->text = $text;
-        $this->startLine = $startLine;
-        $this->startFilePos = $startFilePos;
-        $this->startTokenPos = $startTokenPos;
-        $this->endLine = $endLine;
-        $this->endFilePos = $endFilePos;
-        $this->endTokenPos = $endTokenPos;
+        $this->line = $startLine;
+        $this->filePos = $startFilePos;
+        $this->tokenPos = $startTokenPos;
     }
 
     /**
@@ -46,88 +38,28 @@ class Comment implements \JsonSerializable
     /**
      * Gets the line number the comment started on.
      *
-     * @return int Line number (or -1 if not available)
-     */
-    public function getStartLine() : int {
-        return $this->startLine;
-    }
-
-    /**
-     * Gets the file offset the comment started on.
-     *
-     * @return int File offset (or -1 if not available)
-     */
-    public function getStartFilePos() : int {
-        return $this->startFilePos;
-    }
-
-    /**
-     * Gets the token offset the comment started on.
-     *
-     * @return int Token offset (or -1 if not available)
-     */
-    public function getStartTokenPos() : int {
-        return $this->startTokenPos;
-    }
-
-    /**
-     * Gets the line number the comment ends on.
-     *
-     * @return int Line number (or -1 if not available)
-     */
-    public function getEndLine() : int {
-        return $this->endLine;
-    }
-
-    /**
-     * Gets the file offset the comment ends on.
-     *
-     * @return int File offset (or -1 if not available)
-     */
-    public function getEndFilePos() : int {
-        return $this->endFilePos;
-    }
-
-    /**
-     * Gets the token offset the comment ends on.
-     *
-     * @return int Token offset (or -1 if not available)
-     */
-    public function getEndTokenPos() : int {
-        return $this->endTokenPos;
-    }
-
-    /**
-     * Gets the line number the comment started on.
-     *
-     * @deprecated Use getStartLine() instead
-     *
      * @return int Line number
      */
     public function getLine() : int {
-        return $this->startLine;
+        return $this->line;
     }
 
     /**
      * Gets the file offset the comment started on.
      *
-     * @deprecated Use getStartFilePos() instead
-     *
      * @return int File offset
      */
     public function getFilePos() : int {
-        return $this->startFilePos;
+        return $this->filePos;
     }
 
     /**
      * Gets the token offset the comment started on.
      *
-     * @deprecated Use getStartTokenPos() instead
-     *
      * @return int Token offset
      */
     public function getTokenPos() : int {
-        return $this->startTokenPos;
+        return $this->tokenPos;
     }
 
     /**
@@ -227,13 +159,9 @@ class Comment implements \JsonSerializable
         return [
             'nodeType' => $type,
             'text' => $this->text,
-            // TODO: Rename these to include "start".
-            'line' => $this->startLine,
-            'filePos' => $this->startFilePos,
-            'tokenPos' => $this->startTokenPos,
-            'endLine' => $this->endLine,
-            'endFilePos' => $this->endFilePos,
-            'endTokenPos' => $this->endTokenPos,
+            'line' => $this->line,
+            'filePos' => $this->filePos,
+            'tokenPos' => $this->tokenPos,
         ];
     }
 }

lib/PhpParser/ConstExprEvaluator.php

@@ -2,7 +2,6 @@
 
 namespace PhpParser;
 
-use function array_merge;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Scalar;
 
@@ -151,8 +150,6 @@ class ConstExprEvaluator
         foreach ($expr->items as $item) {
             if (null !== $item->key) {
                 $array[$this->evaluate($item->key)] = $this->evaluate($item->value);
-            } elseif ($item->unpack) {
-                $array = array_merge($array, $this->evaluate($item->value));
             } else {
                 $array[] = $this->evaluate($item->value);
             }

lib/PhpParser/FileContext.php

@@ -0,0 +1,8 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser;
+
+class FileContext {
+    /** @var Token[] */
+    public $tokens;
+}

lib/PhpParser/Internal/PrintableNewAnonClassNode.php

@@ -17,8 +17,6 @@ use PhpParser\Node\Expr;
  */
 class PrintableNewAnonClassNode extends Expr
 {
-    /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
     /** @var Node\Arg[] Arguments */
     public $args;
     /** @var null|Node\Name Name of extended class */
@@ -29,11 +27,9 @@ class PrintableNewAnonClassNode extends Expr
     public $stmts;
 
     public function __construct(
-        array $attrGroups, array $args, Node\Name $extends = null, array $implements,
-        array $stmts, array $attributes
+        array $args, Node\Name $extends = null, array $implements, array $stmts, array $attributes
     ) {
         parent::__construct($attributes);
-        $this->attrGroups = $attrGroups;
         $this->args = $args;
         $this->extends = $extends;
         $this->implements = $implements;
@@ -46,7 +42,7 @@ class PrintableNewAnonClassNode extends Expr
         // We don't assert that $class->name is null here, to allow consumers to assign unique names
         // to anonymous classes for their own purposes. We simplify ignore the name here.
         return new self(
-            $class->attrGroups, $newNode->args, $class->extends, $class->implements,
+            $newNode->args, $class->extends, $class->implements,
             $class->stmts, $newNode->getAttributes()
         );
     }
@@ -56,6 +52,6 @@ class PrintableNewAnonClassNode extends Expr
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'args', 'extends', 'implements', 'stmts'];
+        return ['args', 'extends', 'implements', 'stmts'];
     }
 }

lib/PhpParser/Internal/TokenStream.php

@@ -2,6 +2,8 @@
 
 namespace PhpParser\Internal;
 
+use PhpParser\Token;
+
 /**
  * Provides operations on token streams, for use by pretty printer.
  *
@@ -9,7 +11,7 @@ namespace PhpParser\Internal;
  */
 class TokenStream
 {
-    /** @var array Tokens (in token_get_all format) */
+    /** @var Token[] */
     private $tokens;
     /** @var int[] Map from position to indentation */
     private $indentMap;
@@ -17,7 +19,7 @@ class TokenStream
     /**
      * Create token stream instance.
      *
-     * @param array $tokens Tokens in token_get_all() format
+     * @param Token[] $tokens Tokens
      */
     public function __construct(array $tokens) {
         $this->tokens = $tokens;
@@ -33,8 +35,8 @@ class TokenStream
      * @return bool
      */
     public function haveParens(int $startPos, int $endPos) : bool {
-        return $this->haveTokenImmediatelyBefore($startPos, '(')
-            && $this->haveTokenImmediatelyAfter($endPos, ')');
+        return $this->haveTokenImmediativelyBefore($startPos, \ord('('))
+            && $this->haveTokenImmediatelyAfter($endPos, \ord(')'));
     }
 
     /**
@@ -46,9 +48,8 @@ class TokenStream
      * @return bool
      */
     public function haveBraces(int $startPos, int $endPos) : bool {
-        return ($this->haveTokenImmediatelyBefore($startPos, '{')
-                || $this->haveTokenImmediatelyBefore($startPos, T_CURLY_OPEN))
-            && $this->haveTokenImmediatelyAfter($endPos, '}');
+        return $this->haveTokenImmediativelyBefore($startPos, \ord('{'))
+            && $this->haveTokenImmediatelyAfter($endPos, \ord('}'));
     }
 
     /**
@@ -61,11 +62,11 @@ class TokenStream
      *
      * @return bool Whether the expected token was found
      */
-    public function haveTokenImmediatelyBefore(int $pos, $expectedTokenType) : bool {
+    public function haveTokenImmediativelyBefore(int $pos, $expectedTokenType) : bool {
         $tokens = $this->tokens;
         $pos--;
         for (; $pos >= 0; $pos--) {
-            $tokenType = $tokens[$pos][0];
+            $tokenType = $tokens[$pos]->id;
             if ($tokenType === $expectedTokenType) {
                 return true;
             }
@@ -91,7 +92,7 @@ class TokenStream
         $tokens = $this->tokens;
         $pos++;
         for (; $pos < \count($tokens); $pos++) {
-            $tokenType = $tokens[$pos][0];
+            $tokenType = $tokens[$pos]->id;
             if ($tokenType === $expectedTokenType) {
                 return true;
             }
@@ -111,7 +112,7 @@ class TokenStream
             return $pos;
         }
 
-        if ($tokens[$pos][0] !== $skipTokenType) {
+        if ($tokens[$pos]->id !== $skipTokenType) {
             // Shouldn't happen. The skip token MUST be there
             throw new \Exception('Encountered unexpected token');
         }
@@ -128,7 +129,7 @@ class TokenStream
             return $pos;
         }
 
-        if ($tokens[$pos][0] !== $skipTokenType) {
+        if ($tokens[$pos]->id !== $skipTokenType) {
             // Shouldn't happen. The skip token MUST be there
             throw new \Exception('Encountered unexpected token');
         }
@@ -146,7 +147,7 @@ class TokenStream
     public function skipLeftWhitespace(int $pos) {
         $tokens = $this->tokens;
         for (; $pos >= 0; $pos--) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
             if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
@@ -163,7 +164,7 @@ class TokenStream
     public function skipRightWhitespace(int $pos) {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
             if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
@@ -171,10 +172,10 @@ class TokenStream
         return $pos;
     }
 
-    public function findRight(int $pos, $findTokenType) {
+    public function findRight($pos, $findTokenType) {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
             if ($type === $findTokenType) {
                 return $pos;
             }
@@ -182,30 +183,6 @@ class TokenStream
         return -1;
     }
 
-    /**
-     * Whether the given position range contains a certain token type.
-     *
-     * @param int $startPos Starting position (inclusive)
-     * @param int $endPos Ending position (exclusive)
-     * @param int|string $tokenType Token type to look for
-     * @return bool Whether the token occurs in the given range
-     */
-    public function haveTokenInRange(int $startPos, int $endPos, $tokenType) {
-        $tokens = $this->tokens;
-        for ($pos = $startPos; $pos < $endPos; $pos++) {
-            if ($tokens[$pos][0] === $tokenType) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    public function haveBracesInRange(int $startPos, int $endPos) {
-        return $this->haveTokenInRange($startPos, $endPos, '{')
-            || $this->haveTokenInRange($startPos, $endPos, T_CURLY_OPEN)
-            || $this->haveTokenInRange($startPos, $endPos, '}');
-    }
-
     /**
      * Get indentation before token position.
      *
@@ -231,9 +208,8 @@ class TokenStream
         $result = '';
         for ($pos = $from; $pos < $to; $pos++) {
             $token = $tokens[$pos];
-            if (\is_array($token)) {
-                $type = $token[0];
-                $content = $token[1];
+            $type = $token->id;
+            $content = $token->value;
             if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
                 $result .= $content;
             } else {
@@ -246,9 +222,6 @@ class TokenStream
                     $result .= $content;
                 }
             }
-            } else {
-                $result .= $token;
-            }
         }
         return $result;
     }
@@ -264,8 +237,8 @@ class TokenStream
         foreach ($this->tokens as $token) {
             $indentMap[] = $indent;
 
-            if ($token[0] === \T_WHITESPACE) {
-                $content = $token[1];
+            if ($token->id === \T_WHITESPACE) {
+                $content = $token->value;
                 $newlinePos = \strrpos($content, "\n");
                 if (false !== $newlinePos) {
                     $indent = \strlen($content) - $newlinePos - 1;

lib/PhpParser/JsonDecoder.php

@@ -73,9 +73,7 @@ class JsonDecoder
         }
 
         return new $className(
-            $value['text'],
-            $value['line'] ?? -1, $value['filePos'] ?? -1, $value['tokenPos'] ?? -1,
-            $value['endLine'] ?? -1, $value['endFilePos'] ?? -1, $value['endTokenPos'] ?? -1
+            $value['text'], $value['line'] ?? -1, $value['filePos'] ?? -1, $value['tokenPos'] ?? -1
         );
     }
 

lib/PhpParser/Lexer.php

@@ -6,97 +6,132 @@ use PhpParser\Parser\Tokens;
 
 class Lexer
 {
-    protected $code;
-    protected $tokens;
-    protected $pos;
-    protected $line;
-    protected $filePos;
-    protected $prevCloseTagHasNewline;
-
+    /** @var array Map from PHP tokens to PhpParser tokens. */
     protected $tokenMap;
-    protected $dropTokens;
-    protected $identifierTokens;
-
-    private $attributeStartLineUsed;
-    private $attributeEndLineUsed;
-    private $attributeStartTokenPosUsed;
-    private $attributeEndTokenPosUsed;
-    private $attributeStartFilePosUsed;
-    private $attributeEndFilePosUsed;
-    private $attributeCommentsUsed;
 
     /**
      * Creates a Lexer.
      *
-     * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
-     *                       which is an array of attributes to add to the AST nodes. Possible
-     *                       attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
-     *                       'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
-     *                       first three. For more info see getNextToken() docs.
+     * @param array $options Options array. Currently unused.
      */
     public function __construct(array $options = []) {
-        // Create Map from internal tokens to PhpParser tokens.
-        $this->defineCompatibilityTokens();
         $this->tokenMap = $this->createTokenMap();
-        $this->identifierTokens = $this->createIdentifierTokenMap();
-
-        // map of tokens to drop while lexing (the map is only used for isset lookup,
-        // that's why the value is simply set to 1; the value is never actually used.)
-        $this->dropTokens = array_fill_keys(
-            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
-        );
-
-        $defaultAttributes = ['comments', 'startLine', 'endLine'];
-        $usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
-
-        // Create individual boolean properties to make these checks faster.
-        $this->attributeStartLineUsed = isset($usedAttributes['startLine']);
-        $this->attributeEndLineUsed = isset($usedAttributes['endLine']);
-        $this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
-        $this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
-        $this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
-        $this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
-        $this->attributeCommentsUsed = isset($usedAttributes['comments']);
     }
 
     /**
-     * Initializes the lexer for lexing the provided source code.
+     * Get tokens IDs that should be ignored by the parser.
+     *
+     * @return array
+     */
+    public function getIgnorableTokens(): array {
+        return [
+            Tokens::T_WHITESPACE,
+            Tokens::T_COMMENT,
+            Tokens::T_DOC_COMMENT,
+            Tokens::T_OPEN_TAG,
+            Tokens::T_BAD_CHARACTER,
+        ];
+    }
+
+    /**
+     * Get map for token canonicalization.
+     *
+     * @return array
+     */
+    public function getCanonicalizationMap(): array {
+        return [
+            Tokens::T_OPEN_TAG_WITH_ECHO => Tokens::T_ECHO,
+            Tokens::T_CLOSE_TAG => \ord(';'),
+        ];
+    }
+
+    /**
+     * Tokenizes the given PHP code into an array of Tokens.
      *
      * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
      * the getErrors() method.
      *
-     * @param string $code The source code to lex
+     * @param string $code The source code to tokenize
      * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
      *                                        ErrorHandler\Throwing
+     *
+     * @return Token[] Sequence of tokens
      */
-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+    public function tokenize(string $code, ErrorHandler $errorHandler = null) {
         if (null === $errorHandler) {
             $errorHandler = new ErrorHandler\Throwing();
         }
 
-        $this->code = $code; // keep the code around for __halt_compiler() handling
-        $this->pos  = -1;
-        $this->line =  1;
-        $this->filePos = 0;
-
-        // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
-        // This ensures proper composability, because having a newline is the "safe" assumption.
-        $this->prevCloseTagHasNewline = true;
-
         $scream = ini_set('xdebug.scream', '0');
-
-        $this->tokens = @token_get_all($code);
-        $this->postprocessTokens($errorHandler);
-
+        error_clear_last();
+        $rawTokens = @token_get_all($code);
+        $checkForMissingTokens = null !== error_get_last();
         if (false !== $scream) {
             ini_set('xdebug.scream', $scream);
         }
+
+        $tokens = [];
+        $filePos = 0;
+        $line = 1;
+        foreach ($rawTokens as $rawToken) {
+            if (\is_array($rawToken)) {
+                $token = new Token($this->tokenMap[$rawToken[0]], $rawToken[1], $line, $filePos);
+            } elseif (\strlen($rawToken) == 2) {
+                // Bug in token_get_all() when lexing b".
+                $token = new Token(\ord('"'), $rawToken, $line, $filePos);
+            } else {
+                $token = new Token(\ord($rawToken), $rawToken, $line, $filePos);
             }
 
-    private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
+            $value = $token->value;
+            $tokenLen = \strlen($value);
+            if ($checkForMissingTokens && substr($code, $filePos, $tokenLen) !== $value) {
+                // Something is missing, must be an invalid character
+                $nextFilePos = strpos($code, $value, $filePos);
+                $badCharTokens = $this->handleInvalidCharacterRange(
+                    $code, $filePos, $nextFilePos, $line, $errorHandler);
+                $tokens = array_merge($tokens, $badCharTokens);
+                $filePos = (int) $nextFilePos;
+            }
+
+            $tokens[] = $token;
+            $filePos += $tokenLen;
+            $line += substr_count($value, "\n");
+        }
+
+        if ($filePos !== \strlen($code)) {
+            // Invalid characters at the end of the input
+            $badCharTokens = $this->handleInvalidCharacterRange(
+                $code, $filePos, \strlen($code), $line, $errorHandler);
+            $tokens = array_merge($tokens, $badCharTokens);
+        }
+
+        if (\count($tokens) > 0) {
+            // Check for unterminated comment
+            $lastToken = $tokens[\count($tokens) - 1];
+            if ($this->isUnterminatedComment($lastToken)) {
+                $errorHandler->handleError(new Error('Unterminated comment', [
+                    'startLine' => $line - substr_count($lastToken->value, "\n"),
+                    'endLine' => $line,
+                    'startFilePos' => $filePos - \strlen($lastToken->value),
+                    'endFilePos' => $filePos,
+                ]));
+            }
+        }
+
+        // Add an EOF sentinel token
+        // TODO: Should the value be an empty string instead?
+        $tokens[] = new Token(0, "\0", $line, \strlen($code));
+
+        return $tokens;
+    }
+
+    private function handleInvalidCharacterRange(
+        string $code, int $start, int $end, int $line, ErrorHandler $errorHandler
+    ) {
         $tokens = [];
         for ($i = $start; $i < $end; $i++) {
-            $chr = $this->code[$i];
+            $chr = $code[$i];
             if ($chr === "\0") {
                 // PHP cuts error message after null byte, so need special case
                 $errorMsg = 'Unexpected null byte';
@@ -106,7 +141,7 @@ class Lexer
                 );
             }
 
-            $tokens[] = [\T_BAD_CHARACTER, $chr, $line];
+            $tokens[] = new Token(Tokens::T_BAD_CHARACTER, $chr, $line, $i);
             $errorHandler->handleError(new Error($errorMsg, [
                 'startLine' => $line,
                 'endLine' => $line,
@@ -117,444 +152,26 @@ class Lexer
         return $tokens;
     }
 
-    /**
-     * Check whether comment token is unterminated.
-     *
-     * @return bool
-     */
-    private function isUnterminatedComment($token) : bool {
-        return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
-            && substr($token[1], 0, 2) === '/*'
-            && substr($token[1], -2) !== '*/';
+    private function isUnterminatedComment(Token $token): bool {
+        return ($token->id === \T_COMMENT || $token->id === \T_DOC_COMMENT)
+            && substr($token->value, 0, 2) === '/*'
+            && substr($token->value, -2) !== '*/';
     }
 
-    protected function postprocessTokens(ErrorHandler $errorHandler) {
-        // PHP's error handling for token_get_all() is rather bad, so if we want detailed
-        // error information we need to compute it ourselves. Invalid character errors are
-        // detected by finding "gaps" in the token array. Unterminated comments are detected
-        // by checking if a trailing comment has a "*/" at the end.
-        //
-        // Additionally, we perform a number of canonicalizations here:
-        //  * Use the PHP 8.0 comment format, which does not include trailing whitespace anymore.
-        //  * Use PHP 8.0 T_NAME_* tokens.
-        //  * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
-        //    T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
-
-        $filePos = 0;
-        $line = 1;
-        $numTokens = \count($this->tokens);
-        for ($i = 0; $i < $numTokens; $i++) {
-            $token = $this->tokens[$i];
-
-            // Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
-            // In this case we only need to emit an error.
-            if ($token[0] === \T_BAD_CHARACTER) {
-                $this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
-            }
-
-            if ($token[0] === \T_COMMENT && substr($token[1], 0, 2) !== '/*'
-                    && preg_match('/(\r\n|\n|\r)$/D', $token[1], $matches)) {
-                $trailingNewline = $matches[0];
-                $token[1] = substr($token[1], 0, -strlen($trailingNewline));
-                $this->tokens[$i] = $token;
-                if (isset($this->tokens[$i + 1]) && $this->tokens[$i + 1][0] === \T_WHITESPACE) {
-                    // Move trailing newline into following T_WHITESPACE token, if it already exists.
-                    $this->tokens[$i + 1][1] = $trailingNewline . $this->tokens[$i + 1][1];
-                    $this->tokens[$i + 1][2]--;
-                } else {
-                    // Otherwise, we need to create a new T_WHITESPACE token.
-                    array_splice($this->tokens, $i + 1, 0, [
-                        [\T_WHITESPACE, $trailingNewline, $line],
-                    ]);
-                    $numTokens++;
-                }
-            }
-
-            // Emulate PHP 8 T_NAME_* tokens, by combining sequences of T_NS_SEPARATOR and T_STRING
-            // into a single token.
-            if (\is_array($token)
-                    && ($token[0] === \T_NS_SEPARATOR || isset($this->identifierTokens[$token[0]]))) {
-                $lastWasSeparator = $token[0] === \T_NS_SEPARATOR;
-                $text = $token[1];
-                for ($j = $i + 1; isset($this->tokens[$j]); $j++) {
-                    if ($lastWasSeparator) {
-                        if (!isset($this->identifierTokens[$this->tokens[$j][0]])) {
-                            break;
-                        }
-                        $lastWasSeparator = false;
-                    } else {
-                        if ($this->tokens[$j][0] !== \T_NS_SEPARATOR) {
-                            break;
-                        }
-                        $lastWasSeparator = true;
-                    }
-                    $text .= $this->tokens[$j][1];
-                }
-                if ($lastWasSeparator) {
-                    // Trailing separator is not part of the name.
-                    $j--;
-                    $text = substr($text, 0, -1);
-                }
-                if ($j > $i + 1) {
-                    if ($token[0] === \T_NS_SEPARATOR) {
-                        $type = \T_NAME_FULLY_QUALIFIED;
-                    } else if ($token[0] === \T_NAMESPACE) {
-                        $type = \T_NAME_RELATIVE;
-                    } else {
-                        $type = \T_NAME_QUALIFIED;
-                    }
-                    $token = [$type, $text, $line];
-                    array_splice($this->tokens, $i, $j - $i, [$token]);
-                    $numTokens -= $j - $i - 1;
-                }
-            }
-
-            if ($token === '&') {
-                $next = $i + 1;
-                while (isset($this->tokens[$next]) && $this->tokens[$next][0] === \T_WHITESPACE) {
-                    $next++;
-                }
-                $followedByVarOrVarArg = isset($this->tokens[$next]) &&
-                    ($this->tokens[$next][0] === \T_VARIABLE || $this->tokens[$next][0] === \T_ELLIPSIS);
-                $this->tokens[$i] = $token = [
-                    $followedByVarOrVarArg
-                        ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
-                        : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG,
-                    '&',
-                    $line,
-                ];
-            }
-
-            $tokenValue = \is_string($token) ? $token : $token[1];
-            $tokenLen = \strlen($tokenValue);
-
-            if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
-                // Something is missing, must be an invalid character
-                $nextFilePos = strpos($this->code, $tokenValue, $filePos);
-                $badCharTokens = $this->handleInvalidCharacterRange(
-                    $filePos, $nextFilePos, $line, $errorHandler);
-                $filePos = (int) $nextFilePos;
-
-                array_splice($this->tokens, $i, 0, $badCharTokens);
-                $numTokens += \count($badCharTokens);
-                $i += \count($badCharTokens);
-            }
-
-            $filePos += $tokenLen;
-            $line += substr_count($tokenValue, "\n");
-        }
-
-        if ($filePos !== \strlen($this->code)) {
-            if (substr($this->code, $filePos, 2) === '/*') {
-                // Unlike PHP, HHVM will drop unterminated comments entirely
-                $comment = substr($this->code, $filePos);
-                $errorHandler->handleError(new Error('Unterminated comment', [
-                    'startLine' => $line,
-                    'endLine' => $line + substr_count($comment, "\n"),
-                    'startFilePos' => $filePos,
-                    'endFilePos' => $filePos + \strlen($comment),
-                ]));
-
-                // Emulate the PHP behavior
-                $isDocComment = isset($comment[3]) && $comment[3] === '*';
-                $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
-            } else {
-                // Invalid characters at the end of the input
-                $badCharTokens = $this->handleInvalidCharacterRange(
-                    $filePos, \strlen($this->code), $line, $errorHandler);
-                $this->tokens = array_merge($this->tokens, $badCharTokens);
-            }
-            return;
-        }
-
-        if (count($this->tokens) > 0) {
-            // Check for unterminated comment
-            $lastToken = $this->tokens[count($this->tokens) - 1];
-            if ($this->isUnterminatedComment($lastToken)) {
-                $errorHandler->handleError(new Error('Unterminated comment', [
-                    'startLine' => $line - substr_count($lastToken[1], "\n"),
-                    'endLine' => $line,
-                    'startFilePos' => $filePos - \strlen($lastToken[1]),
-                    'endFilePos' => $filePos,
-                ]));
-            }
-        }
-    }
-
-    /**
-     * Fetches the next token.
-     *
-     * The available attributes are determined by the 'usedAttributes' option, which can
-     * be specified in the constructor. The following attributes are supported:
-     *
-     *  * 'comments'      => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
-     *                       representing all comments that occurred between the previous
-     *                       non-discarded token and the current one.
-     *  * 'startLine'     => Line in which the node starts.
-     *  * 'endLine'       => Line in which the node ends.
-     *  * 'startTokenPos' => Offset into the token array of the first token in the node.
-     *  * 'endTokenPos'   => Offset into the token array of the last token in the node.
-     *  * 'startFilePos'  => Offset into the code string of the first character that is part of the node.
-     *  * 'endFilePos'    => Offset into the code string of the last character that is part of the node.
-     *
-     * @param mixed $value           Variable to store token content in
-     * @param mixed $startAttributes Variable to store start attributes in
-     * @param mixed $endAttributes   Variable to store end attributes in
-     *
-     * @return int Token id
-     */
-    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
-        $startAttributes = [];
-        $endAttributes   = [];
-
-        while (1) {
-            if (isset($this->tokens[++$this->pos])) {
-                $token = $this->tokens[$this->pos];
-            } else {
-                // EOF token with ID 0
-                $token = "\0";
-            }
-
-            if ($this->attributeStartLineUsed) {
-                $startAttributes['startLine'] = $this->line;
-            }
-            if ($this->attributeStartTokenPosUsed) {
-                $startAttributes['startTokenPos'] = $this->pos;
-            }
-            if ($this->attributeStartFilePosUsed) {
-                $startAttributes['startFilePos'] = $this->filePos;
-            }
-
-            if (\is_string($token)) {
-                $value = $token;
-                if (isset($token[1])) {
-                    // bug in token_get_all
-                    $this->filePos += 2;
-                    $id = ord('"');
-                } else {
-                    $this->filePos += 1;
-                    $id = ord($token);
-                }
-            } elseif (!isset($this->dropTokens[$token[0]])) {
-                $value = $token[1];
-                $id = $this->tokenMap[$token[0]];
-                if (\T_CLOSE_TAG === $token[0]) {
-                    $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n")
-                        || false !== strpos($token[1], "\r");
-                } elseif (\T_INLINE_HTML === $token[0]) {
-                    $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
-                }
-
-                $this->line += substr_count($value, "\n");
-                $this->filePos += \strlen($value);
-            } else {
-                $origLine = $this->line;
-                $origFilePos = $this->filePos;
-                $this->line += substr_count($token[1], "\n");
-                $this->filePos += \strlen($token[1]);
-
-                if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
-                    if ($this->attributeCommentsUsed) {
-                        $comment = \T_DOC_COMMENT === $token[0]
-                            ? new Comment\Doc($token[1],
-                                $origLine, $origFilePos, $this->pos,
-                                $this->line, $this->filePos - 1, $this->pos)
-                            : new Comment($token[1],
-                                $origLine, $origFilePos, $this->pos,
-                                $this->line, $this->filePos - 1, $this->pos);
-                        $startAttributes['comments'][] = $comment;
-                    }
-                }
-                continue;
-            }
-
-            if ($this->attributeEndLineUsed) {
-                $endAttributes['endLine'] = $this->line;
-            }
-            if ($this->attributeEndTokenPosUsed) {
-                $endAttributes['endTokenPos'] = $this->pos;
-            }
-            if ($this->attributeEndFilePosUsed) {
-                $endAttributes['endFilePos'] = $this->filePos - 1;
-            }
-
-            return $id;
-        }
-
-        throw new \RuntimeException('Reached end of lexer loop');
-    }
-
-    /**
-     * Returns the token array for current code.
-     *
-     * The token array is in the same format as provided by the
-     * token_get_all() function and does not discard tokens (i.e.
-     * whitespace and comments are included). The token position
-     * attributes are against this token array.
-     *
-     * @return array Array of tokens in token_get_all() format
-     */
-    public function getTokens() : array {
-        return $this->tokens;
-    }
-
-    /**
-     * Handles __halt_compiler() by returning the text after it.
-     *
-     * @return string Remaining text
-     */
-    public function handleHaltCompiler() : string {
-        // text after T_HALT_COMPILER, still including ();
-        $textAfter = substr($this->code, $this->filePos);
-
-        // ensure that it is followed by ();
-        // this simplifies the situation, by not allowing any comments
-        // in between of the tokens.
-        if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
-            throw new Error('__HALT_COMPILER must be followed by "();"');
-        }
-
-        // prevent the lexer from returning any further tokens
-        $this->pos = count($this->tokens);
-
-        // return with (); removed
-        return substr($textAfter, strlen($matches[0]));
-    }
-
-    private function defineCompatibilityTokens() {
-        static $compatTokensDefined = false;
-        if ($compatTokensDefined) {
-            return;
-        }
-
-        $compatTokens = [
-            // PHP 7.4
-            'T_BAD_CHARACTER',
-            'T_FN',
-            'T_COALESCE_EQUAL',
-            // PHP 8.0
-            'T_NAME_QUALIFIED',
-            'T_NAME_FULLY_QUALIFIED',
-            'T_NAME_RELATIVE',
-            'T_MATCH',
-            'T_NULLSAFE_OBJECT_OPERATOR',
-            'T_ATTRIBUTE',
-            // PHP 8.1
-            'T_ENUM',
-            'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
-            'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
-            'T_READONLY',
-        ];
-
-        // PHP-Parser might be used together with another library that also emulates some or all
-        // of these tokens. Perform a sanity-check that all already defined tokens have been
-        // assigned a unique ID.
-        $usedTokenIds = [];
-        foreach ($compatTokens as $token) {
-            if (\defined($token)) {
-                $tokenId = \constant($token);
-                $clashingToken = $usedTokenIds[$tokenId] ?? null;
-                if ($clashingToken !== null) {
-                    throw new \Error(sprintf(
-                        'Token %s has same ID as token %s, ' .
-                        'you may be using a library with broken token emulation',
-                        $token, $clashingToken
-                    ));
-                }
-                $usedTokenIds[$tokenId] = $token;
-            }
-        }
-
-        // Now define any tokens that have not yet been emulated. Try to assign IDs from -1
-        // downwards, but skip any IDs that may already be in use.
-        $newTokenId = -1;
-        foreach ($compatTokens as $token) {
-            if (!\defined($token)) {
-                while (isset($usedTokenIds[$newTokenId])) {
-                    $newTokenId--;
-                }
-                \define($token, $newTokenId);
-                $newTokenId--;
-            }
-        }
-
-        $compatTokensDefined = true;
-    }
-
-    /**
-     * Creates the token map.
-     *
-     * The token map maps the PHP internal token identifiers
-     * to the identifiers used by the Parser. Additionally it
-     * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
-     *
-     * @return array The token map
-     */
-    protected function createTokenMap() : array {
+    private function createTokenMap(): array {
         $tokenMap = [];
 
-        // 256 is the minimum possible token number, as everything below
-        // it is an ASCII value
         for ($i = 256; $i < 1000; ++$i) {
-            if (\T_DOUBLE_COLON === $i) {
-                // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
-                $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
-            } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
-                // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
-                $tokenMap[$i] = Tokens::T_ECHO;
-            } elseif(\T_CLOSE_TAG === $i) {
-                // T_CLOSE_TAG is equivalent to ';'
-                $tokenMap[$i] = ord(';');
-            } elseif ('UNKNOWN' !== $name = token_name($i)) {
-                if ('T_HASHBANG' === $name) {
-                    // HHVM uses a special token for #! hashbang lines
-                    $tokenMap[$i] = Tokens::T_INLINE_HTML;
-                } elseif (defined($name = Tokens::class . '::' . $name)) {
-                    // Other tokens can be mapped directly
-                    $tokenMap[$i] = constant($name);
+            $name = token_name($i);
+            if ('UNKNOWN' === $name) {
+                continue;
+            }
+            $constName = Tokens::class . '::' . $name;
+            if (defined($constName)) {
+                $tokenMap[$i] = constant($constName);
             }
         }
-        }
-
-        // HHVM uses a special token for numbers that overflow to double
-        if (defined('T_ONUMBER')) {
-            $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
-        }
-        // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
-        if (defined('T_COMPILER_HALT_OFFSET')) {
-            $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
-        }
-
-        // Assign tokens for which we define compatibility constants, as token_name() does not know them.
-        $tokenMap[\T_FN] = Tokens::T_FN;
-        $tokenMap[\T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
-        $tokenMap[\T_NAME_QUALIFIED] = Tokens::T_NAME_QUALIFIED;
-        $tokenMap[\T_NAME_FULLY_QUALIFIED] = Tokens::T_NAME_FULLY_QUALIFIED;
-        $tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
-        $tokenMap[\T_MATCH] = Tokens::T_MATCH;
-        $tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR;
-        $tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE;
-        $tokenMap[\T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
-        $tokenMap[\T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG;
-        $tokenMap[\T_ENUM] = Tokens::T_ENUM;
-        $tokenMap[\T_READONLY] = Tokens::T_READONLY;
 
         return $tokenMap;
     }
-
-    private function createIdentifierTokenMap(): array {
-        // Based on semi_reserved production.
-        return array_fill_keys([
-            \T_STRING,
-            \T_STATIC, \T_ABSTRACT, \T_FINAL, \T_PRIVATE, \T_PROTECTED, \T_PUBLIC, \T_READONLY,
-            \T_INCLUDE, \T_INCLUDE_ONCE, \T_EVAL, \T_REQUIRE, \T_REQUIRE_ONCE, \T_LOGICAL_OR, \T_LOGICAL_XOR, \T_LOGICAL_AND,
-            \T_INSTANCEOF, \T_NEW, \T_CLONE, \T_EXIT, \T_IF, \T_ELSEIF, \T_ELSE, \T_ENDIF, \T_ECHO, \T_DO, \T_WHILE,
-            \T_ENDWHILE, \T_FOR, \T_ENDFOR, \T_FOREACH, \T_ENDFOREACH, \T_DECLARE, \T_ENDDECLARE, \T_AS, \T_TRY, \T_CATCH,
-            \T_FINALLY, \T_THROW, \T_USE, \T_INSTEADOF, \T_GLOBAL, \T_VAR, \T_UNSET, \T_ISSET, \T_EMPTY, \T_CONTINUE, \T_GOTO,
-            \T_FUNCTION, \T_CONST, \T_RETURN, \T_PRINT, \T_YIELD, \T_LIST, \T_SWITCH, \T_ENDSWITCH, \T_CASE, \T_DEFAULT,
-            \T_BREAK, \T_ARRAY, \T_CALLABLE, \T_EXTENDS, \T_IMPLEMENTS, \T_NAMESPACE, \T_TRAIT, \T_INTERFACE, \T_CLASS,
-            \T_CLASS_C, \T_TRAIT_C, \T_FUNC_C, \T_METHOD_C, \T_LINE, \T_FILE, \T_DIR, \T_NS_C, \T_HALT_COMPILER, \T_FN,
-            \T_MATCH,
-        ], true);
-    }
 }

lib/PhpParser/Lexer/Emulative.php

@@ -5,92 +5,55 @@ namespace PhpParser\Lexer;
 use PhpParser\Error;
 use PhpParser\ErrorHandler;
 use PhpParser\Lexer;
-use PhpParser\Lexer\TokenEmulator\AttributeEmulator;
-use PhpParser\Lexer\TokenEmulator\EnumTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
-use PhpParser\Lexer\TokenEmulator\ExplicitOctalEmulator;
-use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator;
 use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
-use PhpParser\Lexer\TokenEmulator\MatchTokenEmulator;
-use PhpParser\Lexer\TokenEmulator\NullsafeTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
-use PhpParser\Lexer\TokenEmulator\ReadonlyTokenEmulator;
-use PhpParser\Lexer\TokenEmulator\ReverseEmulator;
-use PhpParser\Lexer\TokenEmulator\TokenEmulator;
+use PhpParser\Lexer\TokenEmulator\TokenEmulatorInterface;
+use PhpParser\Parser\Tokens;
 
 class Emulative extends Lexer
 {
-    const PHP_7_3 = '7.3dev';
-    const PHP_7_4 = '7.4dev';
-    const PHP_8_0 = '8.0dev';
-    const PHP_8_1 = '8.1dev';
+    const PHP_7_3 = '7.3.0dev';
+    const PHP_7_4 = '7.4.0dev';
+
+    const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
+/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
+(?:.*\r?\n)*?
+(?<indentation>\h*)\2(?![a-zA-Z_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
+REGEX;
 
     /** @var mixed[] Patches used to reverse changes introduced in the code */
     private $patches = [];
 
-    /** @var TokenEmulator[] */
-    private $emulators = [];
-
-    /** @var string */
-    private $targetPhpVersion;
+    /** @var TokenEmulatorInterface[] */
+    private $tokenEmulators = [];
 
     /**
-     * @param mixed[] $options Lexer options. In addition to the usual options,
-     *                         accepts a 'phpVersion' string that specifies the
-     *                         version to emulate. Defaults to newest supported.
+     * @param mixed[] $options
      */
     public function __construct(array $options = [])
     {
-        $this->targetPhpVersion = $options['phpVersion'] ?? Emulative::PHP_8_1;
-        unset($options['phpVersion']);
-
         parent::__construct($options);
 
-        $emulators = [
-            new FlexibleDocStringEmulator(),
-            new FnTokenEmulator(),
-            new MatchTokenEmulator(),
-            new CoaleseEqualTokenEmulator(),
-            new NumericLiteralSeparatorEmulator(),
-            new NullsafeTokenEmulator(),
-            new AttributeEmulator(),
-            new EnumTokenEmulator(),
-            new ReadonlyTokenEmulator(),
-            new ExplicitOctalEmulator(),
-        ];
-
-        // Collect emulators that are relevant for the PHP version we're running
-        // and the PHP version we're targeting for emulation.
-        foreach ($emulators as $emulator) {
-            $emulatorPhpVersion = $emulator->getPhpVersion();
-            if ($this->isForwardEmulationNeeded($emulatorPhpVersion)) {
-                $this->emulators[] = $emulator;
-            } else if ($this->isReverseEmulationNeeded($emulatorPhpVersion)) {
-                $this->emulators[] = new ReverseEmulator($emulator);
-            }
-        }
-    }
-
-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
-        $emulators = array_filter($this->emulators, function($emulator) use($code) {
-            return $emulator->isEmulationNeeded($code);
-        });
-
-        if (empty($emulators)) {
-            // Nothing to emulate, yay
-            parent::startLexing($code, $errorHandler);
-            return;
+        $this->tokenEmulators[] = new FnTokenEmulator();
+        $this->tokenEmulators[] = new CoaleseEqualTokenEmulator();
+        $this->tokenEmulators[] = new NumericLiteralSeparatorEmulator();
     }
 
+    public function tokenize(string $code, ErrorHandler $errorHandler = null) {
         $this->patches = [];
-        foreach ($emulators as $emulator) {
-            $code = $emulator->preprocessCode($code, $this->patches);
+
+        if ($this->isEmulationNeeded($code) === false) {
+            // Nothing to emulate, yay
+            return parent::tokenize($code, $errorHandler);
         }
 
         $collector = new ErrorHandler\Collecting();
-        parent::startLexing($code, $collector);
-        $this->sortPatches();
-        $this->fixupTokens();
+
+        // 1. emulation of heredoc and nowdoc new syntax
+        $preparedCode = $this->processHeredocNowdoc($code);
+        $tokens = parent::tokenize($preparedCode, $collector);
+        $tokens = $this->fixupTokens($tokens);
 
         $errors = $collector->getErrors();
         if (!empty($errors)) {
@@ -100,34 +63,86 @@ class Emulative extends Lexer
             }
         }
 
-        foreach ($emulators as $emulator) {
-            $this->tokens = $emulator->emulate($code, $this->tokens);
+        // add token emulation
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                $tokens = $emulativeToken->emulate($code, $tokens);
             }
         }
 
-    private function isForwardEmulationNeeded(string $emulatorPhpVersion): bool {
-        return version_compare(\PHP_VERSION, $emulatorPhpVersion, '<')
-            && version_compare($this->targetPhpVersion, $emulatorPhpVersion, '>=');
+        return $tokens;
     }
 
-    private function isReverseEmulationNeeded(string $emulatorPhpVersion): bool {
-        return version_compare(\PHP_VERSION, $emulatorPhpVersion, '>=')
-            && version_compare($this->targetPhpVersion, $emulatorPhpVersion, '<');
-    }
-
-    private function sortPatches()
+    private function isHeredocNowdocEmulationNeeded(string $code): bool
     {
-        // Patches may be contributed by different emulators.
-        // Make sure they are sorted by increasing patch position.
-        usort($this->patches, function($p1, $p2) {
-            return $p1[0] <=> $p2[0];
-        });
+        // skip version where this works without emulation
+        if (version_compare(\PHP_VERSION, self::PHP_7_3, '>=')) {
+            return false;
         }
 
-    private function fixupTokens()
+        return strpos($code, '<<<') !== false;
+    }
+
+    private function processHeredocNowdoc(string $code): string
+    {
+        if ($this->isHeredocNowdocEmulationNeeded($code) === false) {
+            return $code;
+        }
+
+        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
+            // No heredoc/nowdoc found
+            return $code;
+        }
+
+        // Keep track of how much we need to adjust string offsets due to the modifications we
+        // already made
+        $posDelta = 0;
+        foreach ($matches as $match) {
+            $indentation = $match['indentation'][0];
+            $indentationStart = $match['indentation'][1];
+
+            $separator = $match['separator'][0];
+            $separatorStart = $match['separator'][1];
+
+            if ($indentation === '' && $separator !== '') {
+                // Ordinary heredoc/nowdoc
+                continue;
+            }
+
+            if ($indentation !== '') {
+                // Remove indentation
+                $indentationLen = strlen($indentation);
+                $code = substr_replace($code, '', $indentationStart + $posDelta, $indentationLen);
+                $this->patches[] = [$indentationStart + $posDelta, 'add', $indentation];
+                $posDelta -= $indentationLen;
+            }
+
+            if ($separator === '') {
+                // Insert newline as separator
+                $code = substr_replace($code, "\n", $separatorStart + $posDelta, 0);
+                $this->patches[] = [$separatorStart + $posDelta, 'remove', "\n"];
+                $posDelta += 1;
+            }
+        }
+
+        return $code;
+    }
+
+    private function isEmulationNeeded(string $code): bool
+    {
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                return true;
+            }
+        }
+
+        return $this->isHeredocNowdocEmulationNeeded($code);
+    }
+
+    private function fixupTokens(array $tokens): array
     {
         if (\count($this->patches) === 0) {
-            return;
+            return $tokens;
         }
 
         // Load first patch
@@ -137,55 +152,31 @@ class Emulative extends Lexer
 
         // We use a manual loop over the tokens, because we modify the array on the fly
         $pos = 0;
-        for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
-            $token = $this->tokens[$i];
-            if (\is_string($token)) {
-                if ($patchPos === $pos) {
-                    // Only support replacement for string tokens.
-                    assert($patchType === 'replace');
-                    $this->tokens[$i] = $patchText;
-
-                    // Fetch the next patch
-                    $patchIdx++;
-                    if ($patchIdx >= \count($this->patches)) {
-                        // No more patches, we're done
-                        return;
-                    }
-                    list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
-                }
-
-                $pos += \strlen($token);
-                continue;
-            }
-
-            $len = \strlen($token[1]);
+        for ($i = 0, $c = \count($tokens); $i < $c; $i++) {
+            $token = $tokens[$i];
+            $len = \strlen($token->value);
             $posDelta = 0;
             while ($patchPos >= $pos && $patchPos < $pos + $len) {
                 $patchTextLen = \strlen($patchText);
                 if ($patchType === 'remove') {
                     if ($patchPos === $pos && $patchTextLen === $len) {
                         // Remove token entirely
-                        array_splice($this->tokens, $i, 1, []);
+                        array_splice($tokens, $i, 1, []);
                         $i--;
                         $c--;
                     } else {
                         // Remove from token string
-                        $this->tokens[$i][1] = substr_replace(
-                            $token[1], '', $patchPos - $pos + $posDelta, $patchTextLen
+                        $tokens[$i]->value = substr_replace(
+                            $token->value, '', $patchPos - $pos + $posDelta, $patchTextLen
                         );
                         $posDelta -= $patchTextLen;
                     }
                 } elseif ($patchType === 'add') {
                     // Insert into the token string
-                    $this->tokens[$i][1] = substr_replace(
-                        $token[1], $patchText, $patchPos - $pos + $posDelta, 0
+                    $tokens[$i]->value = substr_replace(
+                        $token->value, $patchText, $patchPos - $pos + $posDelta, 0
                     );
                     $posDelta += $patchTextLen;
-                } else if ($patchType === 'replace') {
-                    // Replace inside the token string
-                    $this->tokens[$i][1] = substr_replace(
-                        $token[1], $patchText, $patchPos - $pos + $posDelta, $patchTextLen
-                    );
                 } else {
                     assert(false);
                 }
@@ -194,21 +185,30 @@ class Emulative extends Lexer
                 $patchIdx++;
                 if ($patchIdx >= \count($this->patches)) {
                     // No more patches, we're done
-                    return;
+                    break 2;
                 }
 
                 list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
 
                 // Multiple patches may apply to the same token. Reload the current one to check
                 // If the new patch applies
-                $token = $this->tokens[$i];
+                $token = $tokens[$i];
             }
 
             $pos += $len;
         }
 
-        // A patch did not apply
-        assert(false);
+        // To retain a minimum amount of sanity, recompute lines and offsets in a separate loop.
+        $pos = 0;
+        $line = 1;
+        foreach ($tokens as $token) {
+            $token->filePos = $pos;
+            $token->line = $line;
+            $pos += \strlen($token->value);
+            $line += \substr_count($token->value, "\n");
+        }
+
+        return $tokens;
     }
 
     /**
@@ -232,7 +232,7 @@ class Emulative extends Lexer
                 if ($patchType === 'add') {
                     $posDelta += strlen($patchText);
                     $lineDelta += substr_count($patchText, "\n");
-                } else if ($patchType === 'remove') {
+                } else {
                     $posDelta -= strlen($patchText);
                     $lineDelta -= substr_count($patchText, "\n");
                 }

lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php

@@ -1,56 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\Lexer\Emulative;
-
-final class AttributeEmulator extends TokenEmulator
-{
-    public function getPhpVersion(): string
-    {
-        return Emulative::PHP_8_0;
-    }
-
-    public function isEmulationNeeded(string $code) : bool
-    {
-        return strpos($code, '#[') !== false;
-    }
-
-    public function emulate(string $code, array $tokens): array
-    {
-        // We need to manually iterate and manage a count because we'll change
-        // the tokens array on the way.
-        $line = 1;
-        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            if ($tokens[$i] === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1] === '[') {
-                array_splice($tokens, $i, 2, [
-                    [\T_ATTRIBUTE, '#[', $line]
-                ]);
-                $c--;
-                continue;
-            }
-            if (\is_array($tokens[$i])) {
-                $line += substr_count($tokens[$i][1], "\n");
-            }
-        }
-
-        return $tokens;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array
-    {
-        // TODO
-        return $tokens;
-    }
-
-    public function preprocessCode(string $code, array &$patches): string {
-        $pos = 0;
-        while (false !== $pos = strpos($code, '#[', $pos)) {
-            // Replace #[ with %[
-            $code[$pos] = '%';
-            $patches[] = [$pos, 'replace', '#'];
-            $pos += 2;
-        }
-        return $code;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php

@@ -3,16 +3,18 @@
 namespace PhpParser\Lexer\TokenEmulator;
 
 use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;
 
-final class CoaleseEqualTokenEmulator extends TokenEmulator
+final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
 {
-    public function getPhpVersion(): string
-    {
-        return Emulative::PHP_7_4;
-    }
-
     public function isEmulationNeeded(string $code) : bool
     {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
         return strpos($code, '??=') !== false;
     }
 
@@ -20,28 +22,19 @@ final class CoaleseEqualTokenEmulator extends TokenEmulator
     {
         // We need to manually iterate and manage a count because we'll change
         // the tokens array on the way
-        $line = 1;
         for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
             if (isset($tokens[$i + 1])) {
-                if ($tokens[$i][0] === T_COALESCE && $tokens[$i + 1] === '=') {
+                $token = $tokens[$i];
+                if ($token->id === Tokens::T_COALESCE && $tokens[$i + 1]->value === '=') {
                     array_splice($tokens, $i, 2, [
-                        [\T_COALESCE_EQUAL, '??=', $line]
+                        new Token(Tokens::T_COALESCE_EQUAL, '??=', $token->line, $token->filePos),
                     ]);
                     $c--;
                     continue;
                 }
             }
-            if (\is_array($tokens[$i])) {
-                $line += substr_count($tokens[$i][1], "\n");
-            }
         }
 
         return $tokens;
     }
-
-    public function reverseEmulate(string $code, array $tokens): array
-    {
-        // ??= was not valid code previously, don't bother.
-        return $tokens;
-    }
 }

lib/PhpParser/Lexer/TokenEmulator/EnumTokenEmulator.php

@@ -1,31 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\Lexer\Emulative;
-
-final class EnumTokenEmulator extends KeywordEmulator
-{
-    public function getPhpVersion(): string
-    {
-        return Emulative::PHP_8_1;
-    }
-
-    public function getKeywordString(): string
-    {
-        return 'enum';
-    }
-
-    public function getKeywordToken(): int
-    {
-        return \T_ENUM;
-    }
-
-    protected function isKeywordContext(array $tokens, int $pos): bool
-    {
-        return parent::isKeywordContext($tokens, $pos)
-            && isset($tokens[$pos + 2])
-            && $tokens[$pos + 1][0] === \T_WHITESPACE
-            && $tokens[$pos + 2][0] === \T_STRING;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/ExplicitOctalEmulator.php

@@ -1,44 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\Lexer\Emulative;
-
-class ExplicitOctalEmulator extends TokenEmulator {
-    public function getPhpVersion(): string {
-        return Emulative::PHP_8_1;
-    }
-
-    public function isEmulationNeeded(string $code): bool {
-        return strpos($code, '0o') !== false || strpos($code, '0O') !== false;
-    }
-
-    public function emulate(string $code, array $tokens): array {
-        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            if ($tokens[$i][0] == \T_LNUMBER && $tokens[$i][1] === '0' &&
-                isset($tokens[$i + 1]) && $tokens[$i + 1][0] == \T_STRING &&
-                preg_match('/[oO][0-7]+(?:_[0-7]+)*/', $tokens[$i + 1][1])
-            ) {
-                $tokenKind = $this->resolveIntegerOrFloatToken($tokens[$i + 1][1]);
-                array_splice($tokens, $i, 2, [
-                    [$tokenKind, '0' . $tokens[$i + 1][1], $tokens[$i][2]],
-                ]);
-                $c--;
-            }
-        }
-        return $tokens;
-    }
-
-    private function resolveIntegerOrFloatToken(string $str): int
-    {
-        $str = substr($str, 1);
-        $str = str_replace('_', '', $str);
-        $num = octdec($str);
-        return is_float($num) ? \T_DNUMBER : \T_LNUMBER;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array {
-        // Explicit octals were not legal code previously, don't bother.
-        return $tokens;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/FlexibleDocStringEmulator.php

@@ -1,76 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\Lexer\Emulative;
-
-final class FlexibleDocStringEmulator extends TokenEmulator
-{
-    const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
-/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
-(?:.*\r?\n)*?
-(?<indentation>\h*)\2(?![a-zA-Z0-9_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
-REGEX;
-
-    public function getPhpVersion(): string
-    {
-        return Emulative::PHP_7_3;
-    }
-
-    public function isEmulationNeeded(string $code) : bool
-    {
-        return strpos($code, '<<<') !== false;
-    }
-
-    public function emulate(string $code, array $tokens): array
-    {
-        // Handled by preprocessing + fixup.
-        return $tokens;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array
-    {
-        // Not supported.
-        return $tokens;
-    }
-
-    public function preprocessCode(string $code, array &$patches): string {
-        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
-            // No heredoc/nowdoc found
-            return $code;
-        }
-
-        // Keep track of how much we need to adjust string offsets due to the modifications we
-        // already made
-        $posDelta = 0;
-        foreach ($matches as $match) {
-            $indentation = $match['indentation'][0];
-            $indentationStart = $match['indentation'][1];
-
-            $separator = $match['separator'][0];
-            $separatorStart = $match['separator'][1];
-
-            if ($indentation === '' && $separator !== '') {
-                // Ordinary heredoc/nowdoc
-                continue;
-            }
-
-            if ($indentation !== '') {
-                // Remove indentation
-                $indentationLen = strlen($indentation);
-                $code = substr_replace($code, '', $indentationStart + $posDelta, $indentationLen);
-                $patches[] = [$indentationStart + $posDelta, 'add', $indentation];
-                $posDelta -= $indentationLen;
-            }
-
-            if ($separator === '') {
-                // Insert newline as separator
-                $code = substr_replace($code, "\n", $separatorStart + $posDelta, 0);
-                $patches[] = [$separatorStart + $posDelta, 'remove', "\n"];
-                $posDelta += 1;
-            }
-        }
-
-        return $code;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php

@@ -3,21 +3,54 @@
 namespace PhpParser\Lexer\TokenEmulator;
 
 use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;
 
-final class FnTokenEmulator extends KeywordEmulator
+final class FnTokenEmulator implements TokenEmulatorInterface
 {
-    public function getPhpVersion(): string
+    public function isEmulationNeeded(string $code) : bool
     {
-        return Emulative::PHP_7_4;
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
         }
 
-    public function getKeywordString(): string
-    {
-        return 'fn';
+        return strpos($code, 'fn') !== false;
     }
 
-    public function getKeywordToken(): int
+    public function emulate(string $code, array $tokens): array
     {
-        return \T_FN;
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        foreach ($tokens as $i => $token) {
+            if ($token->id === Tokens::T_STRING && $token->value === 'fn') {
+                $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $i);
+                if ($previousNonSpaceToken !== null
+                        && $previousNonSpaceToken->id === Tokens::T_OBJECT_OPERATOR) {
+                    continue;
+                }
+
+                $token->id = Tokens::T_FN;
+            }
+        }
+
+        return $tokens;
+    }
+
+    /**
+     * @param Token[] $tokens
+     * @return Token|null
+     */
+    private function getPreviousNonSpaceToken(array $tokens, int $start)
+    {
+        for ($i = $start - 1; $i >= 0; --$i) {
+            if ($tokens[$i]->id === Tokens::T_WHITESPACE) {
+                continue;
+            }
+
+            return $tokens[$i];
+        }
+
+        return null;
     }
 }

lib/PhpParser/Lexer/TokenEmulator/KeywordEmulator.php

@@ -1,62 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-abstract class KeywordEmulator extends TokenEmulator
-{
-    abstract function getKeywordString(): string;
-    abstract function getKeywordToken(): int;
-
-    public function isEmulationNeeded(string $code): bool
-    {
-        return strpos(strtolower($code), $this->getKeywordString()) !== false;
-    }
-
-    protected function isKeywordContext(array $tokens, int $pos): bool
-    {
-        $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $pos);
-        return $previousNonSpaceToken === null || $previousNonSpaceToken[0] !== \T_OBJECT_OPERATOR;
-    }
-
-    public function emulate(string $code, array $tokens): array
-    {
-        $keywordString = $this->getKeywordString();
-        foreach ($tokens as $i => $token) {
-            if ($token[0] === T_STRING && strtolower($token[1]) === $keywordString
-                    && $this->isKeywordContext($tokens, $i)) {
-                $tokens[$i][0] = $this->getKeywordToken();
-            }
-        }
-
-        return $tokens;
-    }
-
-    /**
-     * @param mixed[] $tokens
-     * @return mixed[]|null
-     */
-    private function getPreviousNonSpaceToken(array $tokens, int $start)
-    {
-        for ($i = $start - 1; $i >= 0; --$i) {
-            if ($tokens[$i][0] === T_WHITESPACE) {
-                continue;
-            }
-
-            return $tokens[$i];
-        }
-
-        return null;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array
-    {
-        $keywordToken = $this->getKeywordToken();
-        foreach ($tokens as $i => $token) {
-            if ($token[0] === $keywordToken) {
-                $tokens[$i][0] = \T_STRING;
-            }
-        }
-
-        return $tokens;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/MatchTokenEmulator.php

@@ -1,23 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\Lexer\Emulative;
-
-final class MatchTokenEmulator extends KeywordEmulator
-{
-    public function getPhpVersion(): string
-    {
-        return Emulative::PHP_8_0;
-    }
-
-    public function getKeywordString(): string
-    {
-        return 'match';
-    }
-
-    public function getKeywordToken(): int
-    {
-        return \T_MATCH;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/NullsafeTokenEmulator.php

@@ -1,67 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\Lexer\Emulative;
-
-final class NullsafeTokenEmulator extends TokenEmulator
-{
-    public function getPhpVersion(): string
-    {
-        return Emulative::PHP_8_0;
-    }
-
-    public function isEmulationNeeded(string $code): bool
-    {
-        return strpos($code, '?->') !== false;
-    }
-
-    public function emulate(string $code, array $tokens): array
-    {
-        // We need to manually iterate and manage a count because we'll change
-        // the tokens array on the way
-        $line = 1;
-        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            if ($tokens[$i] === '?' && isset($tokens[$i + 1]) && $tokens[$i + 1][0] === \T_OBJECT_OPERATOR) {
-                array_splice($tokens, $i, 2, [
-                    [\T_NULLSAFE_OBJECT_OPERATOR, '?->', $line]
-                ]);
-                $c--;
-                continue;
-            }
-
-            // Handle ?-> inside encapsed string.
-            if ($tokens[$i][0] === \T_ENCAPSED_AND_WHITESPACE && isset($tokens[$i - 1])
-                && $tokens[$i - 1][0] === \T_VARIABLE
-                && preg_match('/^\?->([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)/', $tokens[$i][1], $matches)
-            ) {
-                $replacement = [
-                    [\T_NULLSAFE_OBJECT_OPERATOR, '?->', $line],
-                    [\T_STRING, $matches[1], $line],
-                ];
-                if (\strlen($matches[0]) !== \strlen($tokens[$i][1])) {
-                    $replacement[] = [
-                        \T_ENCAPSED_AND_WHITESPACE,
-                        \substr($tokens[$i][1], \strlen($matches[0])),
-                        $line
-                    ];
-                }
-                array_splice($tokens, $i, 1, $replacement);
-                $c += \count($replacement) - 1;
-                continue;
-            }
-
-            if (\is_array($tokens[$i])) {
-                $line += substr_count($tokens[$i][1], "\n");
-            }
-        }
-
-        return $tokens;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array
-    {
-        // ?-> was not valid code previously, don't bother.
-        return $tokens;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -3,8 +3,10 @@
 namespace PhpParser\Lexer\TokenEmulator;
 
 use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;
 
-final class NumericLiteralSeparatorEmulator extends TokenEmulator
+final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
 {
     const BIN = '(?:0b[01]+(?:_[01]+)*)';
     const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
@@ -14,58 +16,58 @@ final class NumericLiteralSeparatorEmulator extends TokenEmulator
     const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
     const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
 
-    public function getPhpVersion(): string
-    {
-        return Emulative::PHP_7_4;
-    }
-
     public function isEmulationNeeded(string $code) : bool
     {
-        return preg_match('~[0-9]_[0-9]~', $code)
-            || preg_match('~0x[0-9a-f]+_[0-9a-f]~i', $code);
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
         }
 
+        return preg_match('~[0-9a-f]_[0-9a-f]~i', $code) !== false;
+    }
+
+    /**
+     * @param Token[] $tokens
+     * @return Token[]
+     */
     public function emulate(string $code, array $tokens): array
     {
         // We need to manually iterate and manage a count because we'll change
         // the tokens array on the way
-        $codeOffset = 0;
         for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
             $token = $tokens[$i];
-            $tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
+            $tokenLen = \strlen($token->value);
 
-            if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
-                $codeOffset += $tokenLen;
+            if ($token->id !== Tokens::T_LNUMBER && $token->id !== Tokens::T_DNUMBER) {
                 continue;
             }
 
-            $res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $token->filePos);
             assert($res, "No number at number token position");
 
             $match = $matches[0];
             $matchLen = \strlen($match);
             if ($matchLen === $tokenLen) {
                 // Original token already holds the full number.
-                $codeOffset += $tokenLen;
                 continue;
             }
 
             $tokenKind = $this->resolveIntegerOrFloatToken($match);
-            $newTokens = [[$tokenKind, $match, $token[2]]];
+            $newTokens = [new Token($tokenKind, $match, $token->line, $token->filePos)];
 
             $numTokens = 1;
             $len = $tokenLen;
             while ($matchLen > $len) {
                 $nextToken = $tokens[$i + $numTokens];
-                $nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
-                $nextTokenLen = \strlen($nextTokenText);
+                $nextTokenLen = \strlen($nextToken->value);
 
                 $numTokens++;
                 if ($matchLen < $len + $nextTokenLen) {
                     // Split trailing characters into a partial token.
-                    assert(is_array($nextToken), "Partial token should be an array token");
-                    $partialText = substr($nextTokenText, $matchLen - $len);
-                    $newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
+                    $partialText = substr($nextToken->value, $matchLen - $len);
+                    $newTokens[] = new Token(
+                        $nextToken->id, $partialText, $nextToken->line, $nextToken->filePos
+                    );
                     break;
                 }
 
@@ -74,7 +76,6 @@ final class NumericLiteralSeparatorEmulator extends TokenEmulator
 
             array_splice($tokens, $i, $numTokens, $newTokens);
             $c -= $numTokens - \count($newTokens);
-            $codeOffset += $matchLen;
         }
 
         return $tokens;
@@ -94,12 +95,6 @@ final class NumericLiteralSeparatorEmulator extends TokenEmulator
             $num = +$str;
         }
 
-        return is_float($num) ? T_DNUMBER : T_LNUMBER;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array
-    {
-        // Numeric separators were not legal code previously, don't bother.
-        return $tokens;
+        return is_float($num) ? Tokens::T_DNUMBER : Tokens::T_LNUMBER;
     }
 }

lib/PhpParser/Lexer/TokenEmulator/ReadonlyTokenEmulator.php

@@ -1,23 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\Lexer\Emulative;
-
-final class ReadonlyTokenEmulator extends KeywordEmulator
-{
-    public function getPhpVersion(): string
-    {
-        return Emulative::PHP_8_1;
-    }
-
-    public function getKeywordString(): string
-    {
-        return 'readonly';
-    }
-
-    public function getKeywordToken(): int
-    {
-        return \T_READONLY;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/ReverseEmulator.php

@@ -1,36 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-/**
- * Reverses emulation direction of the inner emulator.
- */
-final class ReverseEmulator extends TokenEmulator
-{
-    /** @var TokenEmulator Inner emulator */
-    private $emulator;
-
-    public function __construct(TokenEmulator $emulator) {
-        $this->emulator = $emulator;
-    }
-
-    public function getPhpVersion(): string {
-        return $this->emulator->getPhpVersion();
-    }
-
-    public function isEmulationNeeded(string $code): bool {
-        return $this->emulator->isEmulationNeeded($code);
-    }
-
-    public function emulate(string $code, array $tokens): array {
-        return $this->emulator->reverseEmulate($code, $tokens);
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array {
-        return $this->emulator->emulate($code, $tokens);
-    }
-
-    public function preprocessCode(string $code, array &$patches): string {
-        return $code;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/TokenEmulator.php

@@ -1,25 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-/** @internal */
-abstract class TokenEmulator
-{
-    abstract public function getPhpVersion(): string;
-
-    abstract public function isEmulationNeeded(string $code): bool;
-
-    /**
-     * @return array Modified Tokens
-     */
-    abstract public function emulate(string $code, array $tokens): array;
-
-    /**
-     * @return array Modified Tokens
-     */
-    abstract public function reverseEmulate(string $code, array $tokens): array;
-
-    public function preprocessCode(string $code, array &$patches): string {
-        return $code;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/TokenEmulatorInterface.php

@@ -0,0 +1,17 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Token;
+
+/** @internal */
+interface TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code): bool;
+
+    /**
+     * @param Token[] $tokens
+     * @return Token[]
+     */
+    public function emulate(string $code, array $tokens): array;
+}

lib/PhpParser/Node.php

@@ -95,6 +95,8 @@ interface Node
     /**
      * Gets the doc comment of the node.
      *
+     * The doc comment has to be the last comment associated with the node.
+     *
      * @return null|Comment\Doc Doc comment object or null
      */
     public function getDocComment();

lib/PhpParser/Node/Arg.php

@@ -2,13 +2,10 @@
 
 namespace PhpParser\Node;
 
-use PhpParser\Node\VariadicPlaceholder;
 use PhpParser\NodeAbstract;
 
 class Arg extends NodeAbstract
 {
-    /** @var Identifier|null Parameter name (for named parameters) */
-    public $name;
     /** @var Expr Value to pass */
     public $value;
     /** @var bool Whether to pass by ref */
@@ -23,21 +20,16 @@ class Arg extends NodeAbstract
      * @param bool  $byRef      Whether to pass by ref
      * @param bool  $unpack     Whether to unpack the argument
      * @param array $attributes Additional attributes
-     * @param Identifier|null $name Parameter name (for named parameters)
      */
-    public function __construct(
-        Expr $value, bool $byRef = false, bool $unpack = false, array $attributes = [],
-        Identifier $name = null
-    ) {
+    public function __construct(Expr $value, bool $byRef = false, bool $unpack = false, array $attributes = []) {
         $this->attributes = $attributes;
-        $this->name = $name;
         $this->value = $value;
         $this->byRef = $byRef;
         $this->unpack = $unpack;
     }
 
     public function getSubNodeNames() : array {
-        return ['name', 'value', 'byRef', 'unpack'];
+        return ['value', 'byRef', 'unpack'];
     }
     
     public function getType() : string {

lib/PhpParser/Node/Attribute.php

@@ -1,34 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-use PhpParser\Node;
-use PhpParser\NodeAbstract;
-
-class Attribute extends NodeAbstract
-{
-    /** @var Name Attribute name */
-    public $name;
-
-    /** @var Arg[] Attribute arguments */
-    public $args;
-
-    /**
-     * @param Node\Name $name       Attribute name
-     * @param Arg[]     $args       Attribute arguments
-     * @param array     $attributes Additional node attributes
-     */
-    public function __construct(Name $name, array $args = [], array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->name = $name;
-        $this->args = $args;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['name', 'args'];
-    }
-
-    public function getType() : string {
-        return 'Attribute';
-    }
-}

lib/PhpParser/Node/AttributeGroup.php

@@ -1,29 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-use PhpParser\Node;
-use PhpParser\NodeAbstract;
-
-class AttributeGroup extends NodeAbstract
-{
-    /** @var Attribute[] Attributes */
-    public $attrs;
-
-    /**
-     * @param Attribute[] $attrs PHP attributes
-     * @param array $attributes Additional node attributes
-     */
-    public function __construct(array $attrs, array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->attrs = $attrs;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['attrs'];
-    }
-
-    public function getType() : string {
-        return 'AttributeGroup';
-    }
-}

lib/PhpParser/Node/ComplexType.php

@@ -1,14 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-use PhpParser\NodeAbstract;
-
-/**
- * This is a base class for complex types, including nullable types and union types.
- *
- * It does not provide any shared behavior and exists only for type-checking purposes.
- */
-abstract class ComplexType extends NodeAbstract
-{
-}

lib/PhpParser/Node/Const_.php

@@ -4,6 +4,9 @@ namespace PhpParser\Node;
 
 use PhpParser\NodeAbstract;
 
+/**
+ * @property Name $namespacedName Namespaced name (for class constants, if using NameResolver)
+ */
 class Const_ extends NodeAbstract
 {
     /** @var Identifier Name */
@@ -11,9 +14,6 @@ class Const_ extends NodeAbstract
     /** @var Expr Value */
     public $value;
 
-    /** @var Name Namespaced name (if using NameResolver) */
-    public $namespacedName;
-
     /**
      * Constructs a const node for use in class const and const statements.
      *

lib/PhpParser/Node/Expr/Array_.php

@@ -10,13 +10,13 @@ class Array_ extends Expr
     const KIND_LONG = 1;  // array() syntax
     const KIND_SHORT = 2; // [] syntax
 
-    /** @var (ArrayItem|null)[] Items */
+    /** @var ArrayItem[] Items */
     public $items;
 
     /**
      * Constructs an array node.
      *
-     * @param (ArrayItem|null)[] $items      Items of the array
+     * @param ArrayItem[] $items      Items of the array
      * @param array       $attributes Additional attributes
      */
     public function __construct(array $items = [], array $attributes = []) {

lib/PhpParser/Node/Expr/ArrowFunction.php

@@ -17,13 +17,11 @@ class ArrowFunction extends Expr implements FunctionLike
     /** @var Node\Param[] */
     public $params = [];
 
-    /** @var null|Node\Identifier|Node\Name|Node\ComplexType */
+    /** @var null|Node\Identifier|Node\Name|Node\NullableType */
     public $returnType;
 
     /** @var Expr */
     public $expr;
-    /** @var Node\AttributeGroup[] */
-    public $attrGroups;
 
     /**
      * @param array $subNodes   Array of the following optional subnodes:
@@ -32,7 +30,6 @@ class ArrowFunction extends Expr implements FunctionLike
      *                          'params'     => array() : Parameters
      *                          'returnType' => null    : Return type
      *                          'expr'       => Expr    : Expression body
-     *                          'attrGroups' => array() : PHP attribute groups
      * @param array $attributes Additional attributes
      */
     public function __construct(array $subNodes = [], array $attributes = []) {
@@ -42,12 +39,11 @@ class ArrowFunction extends Expr implements FunctionLike
         $this->params = $subNodes['params'] ?? [];
         $returnType = $subNodes['returnType'] ?? null;
         $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
-        $this->expr = $subNodes['expr'];
-        $this->attrGroups = $subNodes['attrGroups'] ?? [];
+        $this->expr = $subNodes['expr'] ?? null;
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'static', 'byRef', 'params', 'returnType', 'expr'];
+        return ['static', 'byRef', 'params', 'returnType', 'expr'];
     }
 
     public function returnsByRef() : bool {
@@ -62,10 +58,6 @@ class ArrowFunction extends Expr implements FunctionLike
         return $this->returnType;
     }
 
-    public function getAttrGroups() : array {
-        return $this->attrGroups;
-    }
-
     /**
      * @return Node\Stmt\Return_[]
      */

lib/PhpParser/Node/Expr/CallLike.php

@@ -1,39 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node\Expr;
-
-use PhpParser\Node\Arg;
-use PhpParser\Node\Expr;
-use PhpParser\Node\VariadicPlaceholder;
-
-abstract class CallLike extends Expr {
-    /**
-     * Return raw arguments, which may be actual Args, or VariadicPlaceholders for first-class
-     * callables.
-     *
-     * @return array<Arg|VariadicPlaceholder>
-     */
-    abstract public function getRawArgs(): array;
-
-    /**
-     * Returns whether this call expression is actually a first class callable.
-     */
-    public function isFirstClassCallable(): bool {
-        foreach ($this->getRawArgs() as $arg) {
-            if ($arg instanceof VariadicPlaceholder) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Assert that this is not a first-class callable and return only ordinary Args.
-     *
-     * @return Arg[]
-     */
-    public function getArgs(): array {
-        assert(!$this->isFirstClassCallable());
-        return $this->getRawArgs();
-    }
-}

lib/PhpParser/Node/Expr/Closure.php

@@ -16,12 +16,10 @@ class Closure extends Expr implements FunctionLike
     public $params;
     /** @var ClosureUse[] use()s */
     public $uses;
-    /** @var null|Node\Identifier|Node\Name|Node\ComplexType Return type */
+    /** @var null|Node\Identifier|Node\Name|Node\NullableType Return type */
     public $returnType;
     /** @var Node\Stmt[] Statements */
     public $stmts;
-    /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
 
     /**
      * Constructs a lambda function node.
@@ -33,7 +31,6 @@ class Closure extends Expr implements FunctionLike
      *                          'uses'       => array(): use()s
      *                          'returnType' => null   : Return type
      *                          'stmts'      => array(): Statements
-     *                          'attrGroups' => array(): PHP attributes groups
      * @param array $attributes Additional attributes
      */
     public function __construct(array $subNodes = [], array $attributes = []) {
@@ -45,11 +42,10 @@ class Closure extends Expr implements FunctionLike
         $returnType = $subNodes['returnType'] ?? null;
         $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
         $this->stmts = $subNodes['stmts'] ?? [];
-        $this->attrGroups = $subNodes['attrGroups'] ?? [];
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'static', 'byRef', 'params', 'uses', 'returnType', 'stmts'];
+        return ['static', 'byRef', 'params', 'uses', 'returnType', 'stmts'];
     }
 
     public function returnsByRef() : bool {
@@ -69,10 +65,6 @@ class Closure extends Expr implements FunctionLike
         return $this->stmts;
     }
     
-    public function getAttrGroups() : array {
-        return $this->attrGroups;
-    }
-
     public function getType() : string {
         return 'Expr_Closure';
     }

lib/PhpParser/Node/Expr/FuncCall.php

@@ -5,18 +5,18 @@ namespace PhpParser\Node\Expr;
 use PhpParser\Node;
 use PhpParser\Node\Expr;
 
-class FuncCall extends CallLike
+class FuncCall extends Expr
 {
     /** @var Node\Name|Expr Function name */
     public $name;
-    /** @var array<Node\Arg|Node\VariadicPlaceholder> Arguments */
+    /** @var Node\Arg[] Arguments */
     public $args;
 
     /**
      * Constructs a function call node.
      *
      * @param Node\Name|Expr $name       Function name
-     * @param array<Node\Arg|Node\VariadicPlaceholder> $args       Arguments
+     * @param Node\Arg[]     $args       Arguments
      * @param array          $attributes Additional attributes
      */
     public function __construct($name, array $args = [], array $attributes = []) {
@@ -32,8 +32,4 @@ class FuncCall extends CallLike
     public function getType() : string {
         return 'Expr_FuncCall';
     }
-
-    public function getRawArgs(): array {
-        return $this->args;
-    }
 }

lib/PhpParser/Node/Expr/Match_.php

@@ -1,31 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node\Expr;
-
-use PhpParser\Node;
-use PhpParser\Node\MatchArm;
-
-class Match_ extends Node\Expr
-{
-    /** @var Node\Expr */
-    public $cond;
-    /** @var MatchArm[] */
-    public $arms;
-
-    /**
-     * @param MatchArm[] $arms
-     */
-    public function __construct(Node\Expr $cond, array $arms = [], array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->cond = $cond;
-        $this->arms = $arms;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['cond', 'arms'];
-    }
-
-    public function getType() : string {
-        return 'Expr_Match';
-    }
-}

lib/PhpParser/Node/Expr/MethodCall.php

@@ -5,15 +5,14 @@ namespace PhpParser\Node\Expr;
 use PhpParser\Node\Arg;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
-use PhpParser\Node\VariadicPlaceholder;
 
-class MethodCall extends CallLike
+class MethodCall extends Expr
 {
     /** @var Expr Variable holding object */
     public $var;
     /** @var Identifier|Expr Method name */
     public $name;
-    /** @var array<Arg|VariadicPlaceholder> Arguments */
+    /** @var Arg[] Arguments */
     public $args;
 
     /**
@@ -21,7 +20,7 @@ class MethodCall extends CallLike
      *
      * @param Expr                   $var        Variable holding object
      * @param string|Identifier|Expr $name       Method name
-     * @param array<Arg|VariadicPlaceholder> $args       Arguments
+     * @param Arg[]                  $args       Arguments
      * @param array                  $attributes Additional attributes
      */
     public function __construct(Expr $var, $name, array $args = [], array $attributes = []) {
@@ -38,8 +37,4 @@ class MethodCall extends CallLike
     public function getType() : string {
         return 'Expr_MethodCall';
     }
-
-    public function getRawArgs(): array {
-        return $this->args;
-    }
 }

lib/PhpParser/Node/Expr/New_.php

@@ -3,22 +3,20 @@
 namespace PhpParser\Node\Expr;
 
 use PhpParser\Node;
-use PhpParser\Node\Arg;
 use PhpParser\Node\Expr;
-use PhpParser\Node\VariadicPlaceholder;
 
-class New_ extends CallLike
+class New_ extends Expr
 {
     /** @var Node\Name|Expr|Node\Stmt\Class_ Class name */
     public $class;
-    /** @var array<Arg|VariadicPlaceholder> Arguments */
+    /** @var Node\Arg[] Arguments */
     public $args;
 
     /**
      * Constructs a function call node.
      *
      * @param Node\Name|Expr|Node\Stmt\Class_ $class      Class name (or class node for anonymous classes)
-     * @param array<Arg|VariadicPlaceholder>  $args       Arguments
+     * @param Node\Arg[]                      $args       Arguments
      * @param array                           $attributes Additional attributes
      */
     public function __construct($class, array $args = [], array $attributes = []) {
@@ -34,8 +32,4 @@ class New_ extends CallLike
     public function getType() : string {
         return 'Expr_New';
     }
-
-    public function getRawArgs(): array {
-        return $this->args;
-    }
 }

lib/PhpParser/Node/Expr/NullsafeMethodCall.php

@@ -1,45 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node\Expr;
-
-use PhpParser\Node\Arg;
-use PhpParser\Node\Expr;
-use PhpParser\Node\Identifier;
-use PhpParser\Node\VariadicPlaceholder;
-
-class NullsafeMethodCall extends CallLike
-{
-    /** @var Expr Variable holding object */
-    public $var;
-    /** @var Identifier|Expr Method name */
-    public $name;
-    /** @var array<Arg|VariadicPlaceholder> Arguments */
-    public $args;
-
-    /**
-     * Constructs a nullsafe method call node.
-     *
-     * @param Expr                           $var        Variable holding object
-     * @param string|Identifier|Expr         $name       Method name
-     * @param array<Arg|VariadicPlaceholder> $args       Arguments
-     * @param array                          $attributes Additional attributes
-     */
-    public function __construct(Expr $var, $name, array $args = [], array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->var = $var;
-        $this->name = \is_string($name) ? new Identifier($name) : $name;
-        $this->args = $args;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['var', 'name', 'args'];
-    }
-    
-    public function getType() : string {
-        return 'Expr_NullsafeMethodCall';
-    }
-
-    public function getRawArgs(): array {
-        return $this->args;
-    }
-}

lib/PhpParser/Node/Expr/NullsafePropertyFetch.php

@@ -1,35 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node\Expr;
-
-use PhpParser\Node\Expr;
-use PhpParser\Node\Identifier;
-
-class NullsafePropertyFetch extends Expr
-{
-    /** @var Expr Variable holding object */
-    public $var;
-    /** @var Identifier|Expr Property name */
-    public $name;
-
-    /**
-     * Constructs a nullsafe property fetch node.
-     *
-     * @param Expr                   $var        Variable holding object
-     * @param string|Identifier|Expr $name       Property name
-     * @param array                  $attributes Additional attributes
-     */
-    public function __construct(Expr $var, $name, array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->var = $var;
-        $this->name = \is_string($name) ? new Identifier($name) : $name;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['var', 'name'];
-    }
-    
-    public function getType() : string {
-        return 'Expr_NullsafePropertyFetch';
-    }
-}

lib/PhpParser/Node/Expr/StaticCall.php

@@ -3,18 +3,16 @@
 namespace PhpParser\Node\Expr;
 
 use PhpParser\Node;
-use PhpParser\Node\Arg;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
-use PhpParser\Node\VariadicPlaceholder;
 
-class StaticCall extends CallLike
+class StaticCall extends Expr
 {
     /** @var Node\Name|Expr Class name */
     public $class;
     /** @var Identifier|Expr Method name */
     public $name;
-    /** @var array<Arg|VariadicPlaceholder> Arguments */
+    /** @var Node\Arg[] Arguments */
     public $args;
 
     /**
@@ -22,7 +20,7 @@ class StaticCall extends CallLike
      *
      * @param Node\Name|Expr         $class      Class name
      * @param string|Identifier|Expr $name       Method name
-     * @param array<Arg|VariadicPlaceholder> $args       Arguments
+     * @param Node\Arg[]             $args       Arguments
      * @param array                  $attributes Additional attributes
      */
     public function __construct($class, $name, array $args = [], array $attributes = []) {
@@ -39,8 +37,4 @@ class StaticCall extends CallLike
     public function getType() : string {
         return 'Expr_StaticCall';
     }
-
-    public function getRawArgs(): array {
-        return $this->args;
-    }
 }

lib/PhpParser/Node/Expr/Throw_.php

@@ -1,30 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node\Expr;
-
-use PhpParser\Node;
-
-class Throw_ extends Node\Expr
-{
-    /** @var Node\Expr Expression */
-    public $expr;
-
-    /**
-     * Constructs a throw expression node.
-     *
-     * @param Node\Expr $expr       Expression
-     * @param array     $attributes Additional attributes
-     */
-    public function __construct(Node\Expr $expr, array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->expr = $expr;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['expr'];
-    }
-
-    public function getType() : string {
-        return 'Expr_Throw';
-    }
-}

lib/PhpParser/Node/FunctionLike.php

@@ -16,28 +16,21 @@ interface FunctionLike extends Node
     /**
      * List of parameters
      *
-     * @return Param[]
+     * @return Node\Param[]
      */
     public function getParams() : array;
 
     /**
      * Get the declared return type or null
      *
-     * @return null|Identifier|Name|ComplexType
+     * @return null|Identifier|Node\Name|Node\NullableType
      */
     public function getReturnType();
 
     /**
      * The function body
      *
-     * @return Stmt[]|null
+     * @return Node\Stmt[]|null
      */
     public function getStmts();
-
-    /**
-     * Get PHP attribute groups.
-     *
-     * @return AttributeGroup[]
-     */
-    public function getAttrGroups() : array;
 }

lib/PhpParser/Node/IntersectionType.php

@@ -1,30 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-use PhpParser\NodeAbstract;
-
-class IntersectionType extends ComplexType
-{
-    /** @var (Identifier|Name)[] Types */
-    public $types;
-
-    /**
-     * Constructs an intersection type.
-     *
-     * @param (Identifier|Name)[] $types      Types
-     * @param array               $attributes Additional attributes
-     */
-    public function __construct(array $types, array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->types = $types;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['types'];
-    }
-
-    public function getType() : string {
-        return 'IntersectionType';
-    }
-}

lib/PhpParser/Node/MatchArm.php

@@ -1,31 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-use PhpParser\Node;
-use PhpParser\NodeAbstract;
-
-class MatchArm extends NodeAbstract
-{
-    /** @var null|Node\Expr[] */
-    public $conds;
-    /** @var Node\Expr */
-    public $body;
-
-    /**
-     * @param null|Node\Expr[] $conds
-     */
-    public function __construct($conds, Node\Expr $body, array $attributes = []) {
-        $this->conds = $conds;
-        $this->body = $body;
-        $this->attributes = $attributes;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['conds', 'body'];
-    }
-
-    public function getType() : string {
-        return 'MatchArm';
-    }
-}

lib/PhpParser/Node/Name.php

@@ -85,7 +85,7 @@ class Name extends NodeAbstract
     }
 
     /**
-     * Returns a string representation of the name itself, without taking the name type into
+     * Returns a string representation of the name itself, without taking taking the name type into
      * account (e.g., not including a leading backslash for fully qualified names).
      *
      * @return string String representation

lib/PhpParser/Node/NullableType.php

@@ -2,7 +2,9 @@
 
 namespace PhpParser\Node;
 
-class NullableType extends ComplexType
+use PhpParser\NodeAbstract;
+
+class NullableType extends NodeAbstract
 {
     /** @var Identifier|Name Type */
     public $type;

lib/PhpParser/Node/Param.php

@@ -6,7 +6,7 @@ use PhpParser\NodeAbstract;
 
 class Param extends NodeAbstract
 {
-    /** @var null|Identifier|Name|ComplexType Type declaration */
+    /** @var null|Identifier|Name|NullableType Type declaration */
     public $type;
     /** @var bool Whether parameter is passed by reference */
     public $byRef;
@@ -16,29 +16,20 @@ class Param extends NodeAbstract
     public $var;
     /** @var null|Expr Default value */
     public $default;
-    /** @var int */
-    public $flags;
-    /** @var AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
 
     /**
      * Constructs a parameter node.
      *
      * @param Expr\Variable|Expr\Error                 $var        Parameter variable
      * @param null|Expr                                $default    Default value
-     * @param null|string|Identifier|Name|ComplexType $type       Type declaration
+     * @param null|string|Identifier|Name|NullableType $type       Type declaration
      * @param bool                                     $byRef      Whether is passed by reference
      * @param bool                                     $variadic   Whether this is a variadic argument
      * @param array                                    $attributes Additional attributes
-     * @param int                                     $flags      Optional visibility flags
-     * @param AttributeGroup[]                        $attrGroups PHP attribute groups
      */
     public function __construct(
         $var, Expr $default = null, $type = null,
-        bool $byRef = false, bool $variadic = false,
-        array $attributes = [],
-        int $flags = 0,
-        array $attrGroups = []
+        bool $byRef = false, bool $variadic = false, array $attributes = []
     ) {
         $this->attributes = $attributes;
         $this->type = \is_string($type) ? new Identifier($type) : $type;
@@ -46,12 +37,10 @@ class Param extends NodeAbstract
         $this->variadic = $variadic;
         $this->var = $var;
         $this->default = $default;
-        $this->flags = $flags;
-        $this->attrGroups = $attrGroups;
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'flags', 'type', 'byRef', 'variadic', 'var', 'default'];
+        return ['type', 'byRef', 'variadic', 'var', 'default'];
     }
 
     public function getType() : string {

lib/PhpParser/Node/Scalar/LNumber.php

@@ -62,11 +62,6 @@ class LNumber extends Scalar
             throw new Error('Invalid numeric literal', $attributes);
         }
 
-        // Strip optional explicit octal prefix.
-        if ('o' === $str[1] || 'O' === $str[1]) {
-            $str = substr($str, 2);
-        }
-
         // use intval instead of octdec to get proper cutting behavior with malformed numbers
         $attributes['kind'] = LNumber::KIND_OCT;
         return new LNumber(intval($str, 8), $attributes);

lib/PhpParser/Node/Scalar/String_.php

@@ -100,7 +100,7 @@ class String_ extends Scalar
                 if (isset(self::$replacements[$str])) {
                     return self::$replacements[$str];
                 } elseif ('x' === $str[0] || 'X' === $str[0]) {
-                    return chr(hexdec(substr($str, 1)));
+                    return chr(hexdec($str));
                 } elseif ('u' === $str[0]) {
                     return self::codePointToUtf8(hexdec($matches[2]));
                 } else {

lib/PhpParser/Node/Stmt/Catch_.php

@@ -9,7 +9,7 @@ class Catch_ extends Node\Stmt
 {
     /** @var Node\Name[] Types of exceptions to catch */
     public $types;
-    /** @var Expr\Variable|null Variable for exception */
+    /** @var Expr\Variable Variable for exception */
     public $var;
     /** @var Node\Stmt[] Statements */
     public $stmts;
@@ -18,12 +18,12 @@ class Catch_ extends Node\Stmt
      * Constructs a catch node.
      *
      * @param Node\Name[]   $types      Types of exceptions to catch
-     * @param Expr\Variable|null    $var        Variable for exception
+     * @param Expr\Variable $var        Variable for exception
      * @param Node\Stmt[]   $stmts      Statements
      * @param array         $attributes Additional attributes
      */
     public function __construct(
-        array $types, Expr\Variable $var = null, array $stmts = [], array $attributes = []
+        array $types, Expr\Variable $var, array $stmts = [], array $attributes = []
     ) {
         $this->attributes = $attributes;
         $this->types = $types;

lib/PhpParser/Node/Stmt/ClassConst.php

@@ -10,8 +10,6 @@ class ClassConst extends Node\Stmt
     public $flags;
     /** @var Node\Const_[] Constant declarations */
     public $consts;
-    /** @var Node\AttributeGroup[] */
-    public $attrGroups;
 
     /**
      * Constructs a class const list node.
@@ -19,22 +17,15 @@ class ClassConst extends Node\Stmt
      * @param Node\Const_[] $consts     Constant declarations
      * @param int           $flags      Modifiers
      * @param array         $attributes Additional attributes
-     * @param Node\AttributeGroup[] $attrGroups PHP attribute groups
      */
-    public function __construct(
-        array $consts,
-        int $flags = 0,
-        array $attributes = [],
-        array $attrGroups = []
-    ) {
+    public function __construct(array $consts, int $flags = 0, array $attributes = []) {
         $this->attributes = $attributes;
         $this->flags = $flags;
         $this->consts = $consts;
-        $this->attrGroups = $attrGroups;
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'flags', 'consts'];
+        return ['flags', 'consts'];
     }
 
     /**
@@ -65,15 +56,6 @@ class ClassConst extends Node\Stmt
         return (bool) ($this->flags & Class_::MODIFIER_PRIVATE);
     }
     
-    /**
-     * Whether constant is final.
-     *
-     * @return bool
-     */
-    public function isFinal() : bool {
-        return (bool) ($this->flags & Class_::MODIFIER_FINAL);
-    }
-
     public function getType() : string {
         return 'Stmt_ClassConst';
     }

lib/PhpParser/Node/Stmt/ClassLike.php

@@ -4,76 +4,15 @@ namespace PhpParser\Node\Stmt;
 
 use PhpParser\Node;
 
+/**
+ * @property Node\Name $namespacedName Namespaced name (if using NameResolver)
+ */
 abstract class ClassLike extends Node\Stmt
 {
     /** @var Node\Identifier|null Name */
     public $name;
     /** @var Node\Stmt[] Statements */
     public $stmts;
-    /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
-
-    /** @var Node\Name Namespaced name (if using NameResolver) */
-    public $namespacedName;
-
-    /**
-     * @return TraitUse[]
-     */
-    public function getTraitUses() : array {
-        $traitUses = [];
-        foreach ($this->stmts as $stmt) {
-            if ($stmt instanceof TraitUse) {
-                $traitUses[] = $stmt;
-            }
-        }
-        return $traitUses;
-    }
-
-    /**
-     * @return ClassConst[]
-     */
-    public function getConstants() : array {
-        $constants = [];
-        foreach ($this->stmts as $stmt) {
-            if ($stmt instanceof ClassConst) {
-                $constants[] = $stmt;
-            }
-        }
-        return $constants;
-    }
-
-    /**
-     * @return Property[]
-     */
-    public function getProperties() : array {
-        $properties = [];
-        foreach ($this->stmts as $stmt) {
-            if ($stmt instanceof Property) {
-                $properties[] = $stmt;
-            }
-        }
-        return $properties;
-    }
-
-    /**
-     * Gets property with the given name defined directly in this class/interface/trait.
-     *
-     * @param string $name Name of the property
-     *
-     * @return Property|null Property node or null if the property does not exist
-     */
-    public function getProperty(string $name) {
-        foreach ($this->stmts as $stmt) {
-            if ($stmt instanceof Property) {
-                foreach ($stmt->props as $prop) {
-                    if ($prop instanceof PropertyProperty && $name === $prop->name->toString()) {
-                        return $stmt;
-                    }
-                }
-            }
-        }
-        return null;
-    }
 
     /**
      * Gets all methods defined directly in this class/interface/trait

lib/PhpParser/Node/Stmt/ClassMethod.php

@@ -15,12 +15,10 @@ class ClassMethod extends Node\Stmt implements FunctionLike
     public $name;
     /** @var Node\Param[] Parameters */
     public $params;
-    /** @var null|Node\Identifier|Node\Name|Node\ComplexType Return type */
+    /** @var null|Node\Identifier|Node\Name|Node\NullableType Return type */
     public $returnType;
     /** @var Node\Stmt[]|null Statements */
     public $stmts;
-    /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
 
     private static $magicNames = [
         '__construct'  => true,
@@ -50,7 +48,6 @@ class ClassMethod extends Node\Stmt implements FunctionLike
      *                          'params'     => array()        : Parameters
      *                          'returnType' => null           : Return type
      *                          'stmts'      => array()        : Statements
-     *                          'attrGroups' => array()        : PHP attribute groups
      * @param array $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
@@ -62,11 +59,10 @@ class ClassMethod extends Node\Stmt implements FunctionLike
         $returnType = $subNodes['returnType'] ?? null;
         $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
         $this->stmts = array_key_exists('stmts', $subNodes) ? $subNodes['stmts'] : [];
-        $this->attrGroups = $subNodes['attrGroups'] ?? [];
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'flags', 'byRef', 'name', 'params', 'returnType', 'stmts'];
+        return ['flags', 'byRef', 'name', 'params', 'returnType', 'stmts'];
     }
 
     public function returnsByRef() : bool {
@@ -85,10 +81,6 @@ class ClassMethod extends Node\Stmt implements FunctionLike
         return $this->stmts;
     }
 
-    public function getAttrGroups() : array {
-        return $this->attrGroups;
-    }
-
     /**
      * Whether the method is explicitly or implicitly public.
      *

lib/PhpParser/Node/Stmt/Class_.php

@@ -13,7 +13,6 @@ class Class_ extends ClassLike
     const MODIFIER_STATIC    =  8;
     const MODIFIER_ABSTRACT  = 16;
     const MODIFIER_FINAL     = 32;
-    const MODIFIER_READONLY  = 64;
 
     const VISIBILITY_MODIFIER_MASK = 7; // 1 | 2 | 4
 
@@ -33,7 +32,6 @@ class Class_ extends ClassLike
      *                                'extends'    => null   : Name of extended class
      *                                'implements' => array(): Names of implemented interfaces
      *                                'stmts'      => array(): Statements
-     *                                'attrGroups'  => array(): PHP attribute groups
      * @param array       $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
@@ -43,11 +41,10 @@ class Class_ extends ClassLike
         $this->extends = $subNodes['extends'] ?? null;
         $this->implements = $subNodes['implements'] ?? [];
         $this->stmts = $subNodes['stmts'] ?? [];
-        $this->attrGroups = $subNodes['attrGroups'] ?? [];
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'flags', 'name', 'extends', 'implements', 'stmts'];
+        return ['flags', 'name', 'extends', 'implements', 'stmts'];
     }
 
     /**
@@ -97,10 +94,6 @@ class Class_ extends ClassLike
             throw new Error('Multiple final modifiers are not allowed');
         }
 
-        if ($a & self::MODIFIER_READONLY && $b & self::MODIFIER_READONLY) {
-            throw new Error('Multiple readonly modifiers are not allowed');
-        }
-
         if ($a & 48 && $b & 48) {
             throw new Error('Cannot use the final modifier on an abstract class member');
         }

lib/PhpParser/Node/Stmt/EnumCase.php

@@ -1,37 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node\Stmt;
-
-use PhpParser\Node;
-use PhpParser\Node\AttributeGroup;
-
-class EnumCase extends Node\Stmt
-{
-    /** @var Node\Identifier Enum case name */
-    public $name;
-    /** @var Node\Expr|null Enum case expression */
-    public $expr;
-    /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
-
-    /**
-     * @param string|Node\Identifier    $name       Enum case name
-     * @param Node\Expr|null            $expr       Enum case expression
-     * @param AttributeGroup[]          $attrGroups PHP attribute groups
-     * @param array                     $attributes Additional attributes
-     */
-    public function __construct($name, Node\Expr $expr = null, array $attrGroups = [], array $attributes = []) {
-        parent::__construct($attributes);
-        $this->name = \is_string($name) ? new Node\Identifier($name) : $name;
-        $this->expr = $expr;
-        $this->attrGroups = $attrGroups;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['attrGroups', 'name', 'expr'];
-    }
-
-    public function getType() : string {
-        return 'Stmt_EnumCase';
-    }
-}

lib/PhpParser/Node/Stmt/Enum_.php

@@ -1,40 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node\Stmt;
-
-use PhpParser\Node;
-
-class Enum_ extends ClassLike
-{
-    /** @var null|Node\Identifier Scalar Type */
-    public $scalarType;
-    /** @var Node\Name[] Names of implemented interfaces */
-    public $implements;
-
-    /**
-     * @param string|Node\Identifier|null $name       Name
-     * @param array                       $subNodes   Array of the following optional subnodes:
-     *                                                'scalarType'  => null    : Scalar type
-     *                                                'implements'  => array() : Names of implemented interfaces
-     *                                                'stmts'       => array() : Statements
-     *                                                'attrGroups'  => array() : PHP attribute groups
-     * @param array                       $attributes Additional attributes
-     */
-    public function __construct($name, array $subNodes = [], array $attributes = []) {
-        $this->name = \is_string($name) ? new Node\Identifier($name) : $name;
-        $this->scalarType = $subNodes['scalarType'] ?? null;
-        $this->implements = $subNodes['implements'] ?? [];
-        $this->stmts = $subNodes['stmts'] ?? [];
-        $this->attrGroups = $subNodes['attrGroups'] ?? [];
-
-        parent::__construct($attributes);
-    }
-
-    public function getSubNodeNames() : array {
-        return ['attrGroups', 'name', 'scalarType', 'implements', 'stmts'];
-    }
-
-    public function getType() : string {
-        return 'Stmt_Enum';
-    }
-}

lib/PhpParser/Node/Stmt/Function_.php

@@ -5,6 +5,9 @@ namespace PhpParser\Node\Stmt;
 use PhpParser\Node;
 use PhpParser\Node\FunctionLike;
 
+/**
+ * @property Node\Name $namespacedName Namespaced name (if using NameResolver)
+ */
 class Function_ extends Node\Stmt implements FunctionLike
 {
     /** @var bool Whether function returns by reference */
@@ -13,15 +16,10 @@ class Function_ extends Node\Stmt implements FunctionLike
     public $name;
     /** @var Node\Param[] Parameters */
     public $params;
-    /** @var null|Node\Identifier|Node\Name|Node\ComplexType Return type */
+    /** @var null|Node\Identifier|Node\Name|Node\NullableType Return type */
     public $returnType;
     /** @var Node\Stmt[] Statements */
     public $stmts;
-    /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
-
-    /** @var Node\Name Namespaced name (if using NameResolver) */
-    public $namespacedName;
 
     /**
      * Constructs a function node.
@@ -32,7 +30,6 @@ class Function_ extends Node\Stmt implements FunctionLike
      *                           'params'     => array(): Parameters
      *                           'returnType' => null   : Return type
      *                           'stmts'      => array(): Statements
-     *                           'attrGroups' => array(): PHP attribute groups
      * @param array  $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
@@ -43,11 +40,10 @@ class Function_ extends Node\Stmt implements FunctionLike
         $returnType = $subNodes['returnType'] ?? null;
         $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
         $this->stmts = $subNodes['stmts'] ?? [];
-        $this->attrGroups = $subNodes['attrGroups'] ?? [];
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'byRef', 'name', 'params', 'returnType', 'stmts'];
+        return ['byRef', 'name', 'params', 'returnType', 'stmts'];
     }
 
     public function returnsByRef() : bool {
@@ -62,10 +58,6 @@ class Function_ extends Node\Stmt implements FunctionLike
         return $this->returnType;
     }
 
-    public function getAttrGroups() : array {
-        return $this->attrGroups;
-    }
-
     /** @return Node\Stmt[] */
     public function getStmts() : array {
         return $this->stmts;

lib/PhpParser/Node/Stmt/Interface_.php

@@ -16,7 +16,6 @@ class Interface_ extends ClassLike
      * @param array  $subNodes   Array of the following optional subnodes:
      *                           'extends' => array(): Name of extended interfaces
      *                           'stmts'   => array(): Statements
-     *                           'attrGroups' => array(): PHP attribute groups
      * @param array  $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
@@ -24,11 +23,10 @@ class Interface_ extends ClassLike
         $this->name = \is_string($name) ? new Node\Identifier($name) : $name;
         $this->extends = $subNodes['extends'] ?? [];
         $this->stmts = $subNodes['stmts'] ?? [];
-        $this->attrGroups = $subNodes['attrGroups'] ?? [];
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'name', 'extends', 'stmts'];
+        return ['name', 'extends', 'stmts'];
     }
     
     public function getType() : string {

lib/PhpParser/Node/Stmt/Property.php

@@ -3,9 +3,9 @@
 namespace PhpParser\Node\Stmt;
 
 use PhpParser\Node;
-use PhpParser\Node\ComplexType;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
+use PhpParser\Node\NullableType;
 
 class Property extends Node\Stmt
 {
@@ -13,10 +13,8 @@ class Property extends Node\Stmt
     public $flags;
     /** @var PropertyProperty[] Properties */
     public $props;
-    /** @var null|Identifier|Name|ComplexType Type declaration */
+    /** @var null|Identifier|Name|NullableType Type declaration */
     public $type;
-    /** @var Node\AttributeGroup[] PHP attribute groups */
-    public $attrGroups;
 
     /**
      * Constructs a class property list node.
@@ -24,19 +22,17 @@ class Property extends Node\Stmt
      * @param int                                      $flags      Modifiers
      * @param PropertyProperty[]                       $props      Properties
      * @param array                                    $attributes Additional attributes
-     * @param null|string|Identifier|Name|ComplexType $type       Type declaration
-     * @param Node\AttributeGroup[]                   $attrGroups PHP attribute groups
+     * @param null|string|Identifier|Name|NullableType $type       Type declaration
      */
-    public function __construct(int $flags, array $props, array $attributes = [], $type = null, array $attrGroups = []) {
+    public function __construct(int $flags, array $props, array $attributes = [], $type = null) {
         $this->attributes = $attributes;
         $this->flags = $flags;
         $this->props = $props;
         $this->type = \is_string($type) ? new Identifier($type) : $type;
-        $this->attrGroups = $attrGroups;
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'flags', 'type', 'props'];
+        return ['flags', 'type', 'props'];
     }
 
     /**
@@ -76,15 +72,6 @@ class Property extends Node\Stmt
         return (bool) ($this->flags & Class_::MODIFIER_STATIC);
     }
 
-    /**
-     * Whether the property is readonly.
-     *
-     * @return bool
-     */
-    public function isReadonly() : bool {
-        return (bool) ($this->flags & Class_::MODIFIER_READONLY);
-    }
-
     public function getType() : string {
         return 'Stmt_Property';
     }

lib/PhpParser/Node/Stmt/Throw_.php

@@ -10,7 +10,7 @@ class Throw_ extends Node\Stmt
     public $expr;
 
     /**
-     * Constructs a legacy throw statement node.
+     * Constructs a throw node.
      *
      * @param Node\Expr $expr       Expression
      * @param array     $attributes Additional attributes

lib/PhpParser/Node/Stmt/Trait_.php

@@ -12,18 +12,16 @@ class Trait_ extends ClassLike
      * @param string|Node\Identifier $name Name
      * @param array  $subNodes   Array of the following optional subnodes:
      *                           'stmts' => array(): Statements
-     *                           'attrGroups' => array(): PHP attribute groups
      * @param array  $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
         $this->attributes = $attributes;
         $this->name = \is_string($name) ? new Node\Identifier($name) : $name;
         $this->stmts = $subNodes['stmts'] ?? [];
-        $this->attrGroups = $subNodes['attrGroups'] ?? [];
     }
 
     public function getSubNodeNames() : array {
-        return ['attrGroups', 'name', 'stmts'];
+        return ['name', 'stmts'];
     }
     
     public function getType() : string {

lib/PhpParser/Node/Stmt/TryCatch.php

@@ -18,7 +18,7 @@ class TryCatch extends Node\Stmt
      *
      * @param Node\Stmt[]   $stmts      Statements
      * @param Catch_[]      $catches    Catches
-     * @param null|Finally_ $finally    Optional finally node
+     * @param null|Finally_ $finally    Optionaly finally node
      * @param array         $attributes Additional attributes
      */
     public function __construct(array $stmts, array $catches, Finally_ $finally = null, array $attributes = []) {

lib/PhpParser/Node/UnionType.php

@@ -1,28 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-class UnionType extends ComplexType
-{
-    /** @var (Identifier|Name)[] Types */
-    public $types;
-
-    /**
-     * Constructs a union type.
-     *
-     * @param (Identifier|Name)[] $types      Types
-     * @param array               $attributes Additional attributes
-     */
-    public function __construct(array $types, array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->types = $types;
-    }
-
-    public function getSubNodeNames() : array {
-        return ['types'];
-    }
-    
-    public function getType() : string {
-        return 'UnionType';
-    }
-}