Release PHP-Parser 4.19.1

Fixes https://github.com/nikic/PHP-Parser/issues/986.

.gitattributes

@@ -3,15 +3,9 @@
 /grammar         export-ignore
 /test            export-ignore
 /test_old        export-ignore
-/tools           export-ignore
 .editorconfig    export-ignore
 .gitattributes   export-ignore
 .gitignore       export-ignore
-.php-cs-fixer.dist.php  export-ignore
-Makefile         export-ignore
 CHANGELOG.md     export-ignore
-CONTRIBUTING.md  export-ignore
-phpstan-baseline.neon   export-ignore
-phpstan.neon.dist   export-ignore
 phpunit.xml.dist export-ignore
 UPGRADE-*.md     export-ignore

.github/workflows/main.yml

@@ -5,22 +5,22 @@ on:
   pull_request:
 
 jobs:
-  tests_coverage:
+  tests_71:
     runs-on: "ubuntu-latest"
-    name: "PHP 7.4 Unit Tests (with coverage)"
+    name: "PHP 7.1 Unit Tests"
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v4"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
           coverage: "xdebug"
-          php-version: "7.4"
+          php-version: "7.1"
           tools: composer:v2
       - name: "Install dependencies"
         run: |
           composer require php-coveralls/php-coveralls:^2.2 --dev --no-update
-          COMPOSER_ROOT_VERSION=dev-master composer update --no-progress --prefer-dist
+          composer update --no-progress --prefer-dist
       - name: "Tests"
         run: "php vendor/bin/phpunit --coverage-clover build/logs/clover.xml"
       - name: Coveralls
@@ -34,91 +34,55 @@ jobs:
     strategy:
       matrix:
         php-version:
+          - "7.2"
+          - "7.3"
+          - "7.4"
           - "8.0"
           - "8.1"
           - "8.2"
           - "8.3"
-          - "8.4"
-      fail-fast: false
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v4"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
           coverage: "none"
           php-version: "${{ matrix.php-version }}"
-          ini-file: "development"
           tools: composer:v2
       - name: "Install dependencies"
-        run: "COMPOSER_ROOT_VERSION=dev-master composer update --no-progress --prefer-dist ${{ matrix.flags }}"
+        run: "composer update --no-progress --prefer-dist ${{ matrix.flags }}"
       - name: "PHPUnit"
         run: "php vendor/bin/phpunit"
   test_old_73_80:
     runs-on: "ubuntu-latest"
-    name: "PHP 7.4 Code on PHP 8.4 Integration Tests"
+    name: "PHP 7.3 Code on PHP 8.0 Integration Tests"
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v4"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
           coverage: "none"
-          php-version: "8.4"
+          php-version: "8.0"
-          ini-file: "development"
           tools: composer:v2
       - name: "Install PHP 8 dependencies"
-        run: "COMPOSER_ROOT_VERSION=dev-master composer update --no-progress --prefer-dist"
+        run: "composer update --no-progress --prefer-dist"
       - name: "Tests"
-        run: "test_old/run-php-src.sh 7.4.33"
+        run: "test_old/run-php-src.sh 7.3.21"
-  test_old_80_70:
+  test_old_80_71:
     runs-on: "ubuntu-latest"
-    name: "PHP 8.4 Code on PHP 7.4 Integration Tests"
+    name: "PHP 8.1 Code on PHP 7.1 Integration Tests"
     steps:
       - name: "Checkout"
-        uses: "actions/checkout@v4"
+        uses: "actions/checkout@v3"
       - name: "Install PHP"
         uses: "shivammathur/setup-php@v2"
         with:
           coverage: "none"
-          php-version: "7.4"
+          php-version: "7.1"
-          ini-file: "development"
           tools: composer:v2
       - name: "Install PHP 8 dependencies"
-        run: "COMPOSER_ROOT_VERSION=dev-master composer update --no-progress --prefer-dist"
+        run: "composer update --no-progress --prefer-dist"
       - name: "Tests"
-        run: "test_old/run-php-src.sh 8.4.0beta5"
+        run: "test_old/run-php-src.sh 8.1.6"
-  phpstan:
-    runs-on: "ubuntu-latest"
-    name: "PHPStan"
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v4"
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          coverage: "none"
-          php-version: "8.3"
-          tools: composer:v2
-      - name: "Install dependencies"
-        run: |
-          cd tools && composer install
-      - name: "PHPStan"
-        run: "php tools/vendor/bin/phpstan"
-  php-cs-fixer:
-    runs-on: "ubuntu-latest"
-    name: "PHP-CS-Fixer"
-    steps:
-      - name: "Checkout"
-        uses: "actions/checkout@v4"
-      - name: "Install PHP"
-        uses: "shivammathur/setup-php@v2"
-        with:
-          coverage: "none"
-          php-version: "8.3"
-          tools: composer:v2
-      - name: "Install dependencies"
-        run: |
-          cd tools && composer install
-      - name: "php-cs-fixer"
-        run: "php tools/vendor/bin/php-cs-fixer fix --dry-run"

.gitignore

@@ -1,7 +1,5 @@
-.idea/
 vendor/
 composer.lock
 grammar/kmyacc.exe
 grammar/y.output
 .phpunit.result.cache
-.php-cs-fixer.cache

.php-cs-fixer.dist.php

@@ -1,31 +0,0 @@
-<?php
-
-$finder = PhpCsFixer\Finder::create()
-    ->exclude('PhpParser/Parser')
-    ->in(__DIR__ . '/lib')
-    ->in(__DIR__ . '/test')
-    ->in(__DIR__ . '/grammar')
-;
-
-$config = new PhpCsFixer\Config();
-return $config->setRiskyAllowed(true)
-    ->setRules([
-        '@PSR12' => true,
-        // We use PSR12 with consistent brace placement.
-        'curly_braces_position' => [
-            'functions_opening_brace' => 'same_line',
-            'classes_opening_brace' => 'same_line',
-        ],
-        // declare(strict_types=1) on the same line as <?php.
-        'blank_line_after_opening_tag' => false,
-        'declare_strict_types' => true,
-        // Keep argument formatting for now.
-        'method_argument_space' => ['on_multiline' => 'ignore'],
-        'phpdoc_align' => ['align' => 'left'],
-        'phpdoc_trim' => true,
-        'no_empty_phpdoc' => true,
-        'no_superfluous_phpdoc_tags' => ['allow_mixed' => true],
-        'no_extra_blank_lines' => true,
-    ])
-    ->setFinder($finder)
-;

CHANGELOG.md

@@ -1,301 +1,101 @@
-Version 5.5.0 (2025-05-31)
+Version 4.19.1 (2024-03-17)
---------------------------
+---------------------------
-
-### Added
-
-* [8.5] Added support for attributes on constants. `Stmt\Const_` now has an `attrGroups` subnode.
-* Added `weakReferences` option to `NodeConnectingVisitor` and `ParentConnectingVisitor`. This
-  will create the parent/next/prev references as WeakReferences, to avoid making the AST cyclic
-  and thus increasing GC pressure.
-
-### Changed
-
-* Attributes on parameters are now printed on separate lines if the pretty printer target version
-  is PHP 7.4 or older (which is the default). This allows them to be interpreted as comments,
-  instead of causing a parse error. Specify a target version of PHP 8.0 or newer to restore the
-  previous behavior.
-
-Version 5.4.0 (2024-12-30)
---------------------------
-
-### Added
-
-* Added `Property::isAbstract()` and `Property::isFinal()` methods.
-* Added `PropertyHook::isFinal()` method.
-* Emit an error if property hook is used on declaration with multiple properties.
 
 ### Fixed
 
-* Make legacy class aliases compatible with classmap-authoritative autoloader.
+* Fixed "Optional parameter before required parameter" deprecation warning introduced in
-* `Param::isPromoted()` and `Param::isPublic()` now returns true for parameters that have property
+  previous version.
-  hooks but no explicit visibility modifier.
-* `PropertyHook::getStmts()` now correctly desugars short `set` hooks. `set => $value` will be
-  expanded to `set { $this->propertyName = $value; }`. This requires the `propertyName` attribute
-  on the hook to be set, which is now also set by the parser. If the attribute is not set,
-  `getStmts()` will throw an error for short set hooks, as it is not possible to produce a correct
-  desugaring.
 
-Version 5.3.1 (2024-10-08)
+Version 4.19.0 (2024-03-16)
---------------------------
+---------------------------
-
-### Added
-
-* Added support for declaring functions with name `exit` or `die`, to allow their use in stubs.
-
-Version 5.3.0 (2024-09-29)
---------------------------
-
-### Added
-
-* Added `indent` option to pretty printer, which can be used to specify the indentation to use
-  (defaulting to four spaces). This also allows using tab indentation.
-
-### Fixed
-
-* Resolve names in `PropertyHook`s in the `NameResolver`.
-* Include the trailing semicolon inside `Stmt\GroupUse` nodes, making them consistent with
-  `Stmt\Use_` nodes.
-* Fixed indentation sometimes becoming negative in formatting-preserving pretty printer, resulting
-  in `ValueError`s.
-
-Version 5.2.0 (2024-09-15)
---------------------------
-
-### Added
-
-* [8.4] Added support for `__PROPERTY__` magic constant, represented using a
-  `Node\Scalar\MagicConst\Property` node.
-* [8.4] Added support for property hooks, which are represented using a new `hooks` subnode on
-  `Node\Stmt\Property` and `Node\Param`, which contains an array of `Node\PropertyHook`.
-* [8.4] Added support for asymmetric visibility modifiers. Property `flags` can now hold the
-  additional bits `Modifiers::PUBLIC_SET`, `Modifiers::PROTECTED_SET` and `Modifiers::PRIVATE_SET`.
-* [8.4] Added support for generalized exit function. For backwards compatibility, exit without
-  argument or a single plain argument continues to use a `Node\Expr\Exit_` node. Otherwise (e.g.
-  if a named argument is used) it will be represented as a plain `Node\Expr\FuncCall`.
-* Added support for passing enum values to various builder methods, like `BuilderFactory::val()`.
-
-### Removed
-
-* Removed support for alternative array syntax `$array{0}` from the PHP 8 parser. It is still
-  supported by the PHP 7 parser. This is necessary in order to support property hooks.
-
-Version 5.1.0 (2024-07-01)
---------------------------
-
-### Added
-
-* [8.4] Added support for dereferencing `new` expressions without parentheses.
-
-### Fixed
-
-* Fixed redundant parentheses being added when pretty printing ternary expressions.
 
 ### Changed
 
-* Made some phpdoc types more precise.
+* Do not use implicitly nullable parameters, which are deprecated in PHP 8.4.
+* Remove support for running on PHP 7.0, which does not support explicitly nullable parameters.
 
-Version 5.0.2 (2024-03-05)
+Version 4.18.0 (2023-12-10)
---------------------------
+---------------------------
-
-### Fixed
-
-* Fix handling of indentation on next line after opening PHP tag in formatting-preserving pretty
-printer.
-
-### Changed
-
-* Avoid cyclic references in `Parser` objects. This means that no longer used parser objects are
-  immediately destroyed now, instead of requiring cycle GC.
-* Update `PhpVersion::getNewestSupported()` to report PHP 8.3 instead of PHP 8.2.
-
-Version 5.0.1 (2024-02-21)
---------------------------
-
-### Changed
-
-* Added check to detect use of PHP-Parser with libraries that define `T_*` compatibility tokens
-  with incorrect type (such as string instead of int). This would lead to `TypeError`s down the
-  line. Now an `Error` will be thrown early to indicate the problem.
-
-Version 5.0.0 (2024-01-07)
---------------------------
-
-See UPGRADE-5.0 for detailed migration instructions.
-
-### Fixed
-
-* Fixed parent class of `PropertyItem` and `UseItem`.
-
-Version 5.0.0-rc1 (2023-12-20)
-------------------------------
-
-See UPGRADE-5.0 for detailed migration instructions.
-
-### Fixed
-
-* Fixed parsing of empty files.
 
 ### Added
 
-* Added support for printing additional attributes (like `kind`) in `NodeDumper`.
+* Added methods `ParserFactory::createForNewestSupportedVersion()` and
-* Added `rawValue` attribute to `InterpolatedStringPart` and heredoc/nowdoc `String_`s, which
+  `ParserFactory::createForHostVersion()` for forward-compatibility with PHP-Parser 5.0.
-  provides the original, unparsed value. It was previously only available for non-interpolated
+
-  single/double quoted strings.
+### Fixed
-* Added `Stmt\Block` to represent `{}` code blocks. Previously, such code blocks were flattened
+
-  into the parent statements array. `Stmt\Block` will not be created for structures that are
+* Fixed missing name resolution of class constant types.
-  typically used with code blocks, for example `if ($x) { $y; }` will be represented as previously,
+* Fixed class members being dropped if an error is encountered while parsing a later class member
-  while `if ($x) { { $x; } }` will have an extra `Stmt\Block` wrapper.
+  (when error recovery is enabeld).
 
 ### Changed
 
-* Use visitor to assign comments. This fixes the long-standing issue where comments were assigned
+* The `grammar/` directory has been excluded from exported git archives.
-  to all nodes sharing a starting position. Now only the outer-most node will hold the comments.
-* Don't parse unicode escape sequences when targeting PHP < 7.0.
-* Improve NodeDumper performance for large dumps.
 
-### Removed
+Version 4.17.1 (2023-08-13)
+---------------------------
 
-* Removed `Stmt\Throw_` node, use `Expr\Throw_` inside `Stmt\Expression` instead.
+### Fixed
-* Removed `ParserFactory::create()`.
 
-Version 5.0.0-beta1 (2023-09-17)
+* Fixed phpdoc mismatches for `ClassConst::$type` introduced in previous release.
---------------------------------
 
-See UPGRADE-5.0 for detailed migration instructions.
+Version 4.17.0 (2023-08-13)
+---------------------------
 
 ### Added
 
-* Visitors can now be passed directly to the `NodeTraverser` constructor. A separate call to
+* [PHP 8.3] Added support for typed class constants.
-  `addVisitor()` is no longer required.
+* [PHP 8.3] Added supprot for dynamic class constant fetch.
+* [PHP 8.3] Added support for readonly anonymous classes.
 
-### Changed
+### Fixed
 
-* The minimum host PHP version is now PHP 7.4. It is still possible to parse code from older
+* Fixed missing required parentheses when pretty printing new with an expression class name.
-  versions. Property types have been added where possible.
+* Fixed missing required parentheses when pretty printing `(CONST)::$x` and similar.
-* The `Lexer` no longer accepts options. `Lexer\Emulative` only accepts a `PhpVersion`. The
-  `startLexing()`, `getTokens()` and `handleHaltCompiler()` methods have been removed. Instead,
-  there is a single method `tokenize()` returning the tokens.
-* The `Parser::getLexer()` method has been replaced by `Parser::getTokens()`.
-* Attribute handling has been moved from the lexer to the parser, and is no longer configurable.
-  The comments, startLine, endLine, startTokenPos, endTokenPos, startFilePos, and endFilePos
-  attributes will always be added.
-* The pretty printer now defaults to PHP 7.4 as the target version.
-* The pretty printer now indents heredoc/nowdoc strings if the target version is >= 7.3
-  (flexible heredoc/nowdoc).
 
-### Removed
+Version 4.16.0 (2023-06-25)
+---------------------------
 
-* The deprecated `Comment::getLine()`, `Comment::getTokenPos()` and `Comment::getFilePos()` methods
+### Added
-  have been removed. Use `Comment::getStartLine()`, `Comment::getStartTokenPos()` and
+
-  `Comment::getStartFilePos()` instead.
+* Added `Name::getParts()` method for forward-compatibility with PHP-Parser 5.
 
 ### Deprecated
 
-* The `Node::getLine()` method has been deprecated. Use `Node::getStartLine()` instead.
+* Deprecated direct access to `Name::$parts`, which will be removed in PHP-Parser 5.
 
-Version 5.0.0-alpha3 (2023-06-24)
+Version 4.15.5 (2023-05-19)
----------------------------------
+---------------------------
-
-See UPGRADE-5.0 for detailed migration instructions.
 
 ### Added
 
-* [PHP 8.3] Added support for typed constants.
+* Added `makePrivate()`, `makeProtected()`, `makePublic()` and `makeReadonly()` methods to
-* [PHP 8.3] Added support for readonly anonymous classes.
+  `Builder\Param` to allow the creation of promoted parameters.
-* Added support for `NodeVisitor::REPLACE_WITH_NULL`.
-* Added support for CRLF newlines in the pretty printer, using the new `newline` option.
 
-### Changed
+Version 4.15.4 (2023-03-05)
-
+---------------------------
-* Use PHP 7.1 as the default target version for the pretty printer.
-* Print `else if { }` instead of `else { if { } }`.
-* The `leaveNode()` method on visitors is now invoked in reverse order of `enterNode()`.
-* Moved `NodeTraverser::REMOVE_NODE` etc. to `NodeVisitor::REMOVE_NODE`. The old constants are still
-  available for compatibility.
-* The `Name` subnode `parts` has been replaced by `name`, which stores the name as a string rather
-  than an array of parts separated by namespace separators. The `getParts()` method returns the old
-  representation.
-* No longer accept strings for types in Node constructors. Instead, either an `Identifier`, `Name`
-  or `ComplexType` must be passed.
-* `Comment::getReformattedText()` now normalizes CRLF newlines to LF newlines.
 
 ### Fixed
 
-* Don't trim leading whitespace in formatting preserving printer.
+* Fixed formatting-preservation for alternative if syntax with trailing comments.
-* Treat DEL as a label character in the formatting preserving printer depending on the targeted
-  PHP version.
-* Fix error reporting in emulative lexer without explicitly specified error handler.
-* Gracefully handle non-contiguous array indices in the `Differ`.
 
-Version 5.0.0-alpha2 (2023-03-05)
+Version 4.15.3 (2023-01-16)
----------------------------------
+---------------------------
-
-See UPGRADE-5.0 for detailed migration instructions.
-
-### Added
-
-* [PHP 8.3] Added support for dynamic class constant fetch.
-* Added many additional type annotations. PhpStan is now used.
-* Added a fuzzing target for PHP-Fuzzer, which was how a lot of pretty printer bugs were found.
-* Added `isPromoted()`, `isPublic()`, `isProtected()`, `isPrivate()` and `isReadonly()` methods
-  on `Param`.
-* Added support for class constants in trait builder.
-* Added `PrettyPrinter` interface.
-* Added support for formatting preservation when toggling static modifiers.
-* The `php-parse` binary now accepts `-` as the file name, in which case it will read from stdin.
 
 ### Fixed
 
-* The pretty printer now uses a more accurate treatment of unary operator precedence, and will only
+* Support readonly property with PHP 8.2 DNF type.
-  wrap them in parentheses if required. This allowed fixing a number of other precedence related
+* Fixed PHP attribute group and PHP-Parser attribute mixup in EnumCase builder.
-  bugs.
-* The pretty printer now respects the precedence of `clone`, `throw` and arrow functions.
-* The pretty printer no longer unconditionally wraps `yield` in parentheses, unless the target
-  version is set to older than PHP 7.0.
-* Fixed formatting preservation for alternative elseif/else syntax.
-* Fixed checks for when it is safe to print strings as heredoc/nowdoc to accommodate flexible
-  doc string semantics.
-* The pretty printer now prints parentheses around new/instanceof operands in all required
-  situations.
-* Similar, differences in allowed expressions on the LHS of `->` and `::` are now taken into account.
-* Fixed various cases where `\r` at the end of a doc string could be incorrectly merged into a CRLF
-  sequence with a following `\n`.
-* `__halt_compiler` is no longer recognized as a semi-reserved keyword, in line with PHP behavior.
-* `<?=` is no longer recognized as a semi-reserved keyword.
-* Fix handling of very large overflowing `\u` escape sequences.
 
-### Removed
+Version 4.15.2 (2022-11-12)
+---------------------------
 
-* Removed deprecated `Error` constructor taking a line number instead of an attributes array.
+### Fixed
 
-Version 5.0.0-alpha1 (2022-09-04)
+* Fixed parsing of large hex float literals that contain an "e" character.
----------------------------------
+* Fixed tests to pass on 32-bit.
-
+* Fixed generation of invalid code when using formatting-preserving pretty printer with code that
-See UPGRADE-5.0 for detailed migration instructions.
+  uses inline HTML.
-
-### Changed
-
-* PHP 7.1 is now required to run PHP-Parser.
-* Formatting of the standard pretty printer has been adjusted to match PSR-12 more closely.
-* The internal token representation now uses a `PhpParser\Token` class, which is compatible with
-  PHP 8 token representation (`PhpToken`).
-* Destructuring is now always represented using `Expr\List_` nodes, even if it uses `[]` syntax.
-* Renamed a number of node classes, and moved things that were not real expressions/statements
-  outside the `Expr`/`Stmt` hierarchy. Compatibility shims for the old names have been retained.
-
-### Added
-
-* Added `PhpVersion` class, which is accepted in a number of places (e.g. ParserFactory, Parser,
-  Lexer, PrettyPrinter) and gives more precise control over the PHP version being targeted.
-* Added PHP 8 parser though it only differs from the PHP 7 parser in concatenation precedence.
-* Added `Parser::getLexer()` method.
-* Added a `Modifiers` class, as a replacement for `Stmt\Class_::MODIFIER_*`.
-* Added support for returning an array or `REMOVE_NODE` from `NodeVisitor::enterNode()`.
-
-### Removed
-
-* The PHP 5 parser has been removed. The PHP 7 parser has been adjusted to deal with PHP 5 code
-  more gracefully.
 
 Version 4.15.1 (2022-09-04)
 ---------------------------

CONTRIBUTING.md

@@ -1,32 +0,0 @@
-## Coding Style
-
-This project uses PSR-12 with consistent brace placement. This means that the opening brace is
-always on the same line, even for class and method declarations.
-
-## Tools
-
-This project uses PHP-CS-Fixer and PHPStan. You can invoke them using `make`:
-
-```shell
-make php-cs-fixer
-make phpstan
-```
-
-## Adding support for new PHP syntax
-
-1. If necessary, add emulation support for new tokens.
-   * Add a new subclass of `Lexer\TokenEmulator`. Take inspiration from existing classes.
-   * Add the new class to the array in `Lexer\Emulative`.
-   * Add tests for the emulation in `Lexer\EmulativeTest`. You'll want to modify
-     `provideTestReplaceKeywords()` for new reserved keywords and `provideTestLexNewFeatures()` for
-     other emulations.
-2. Add any new node classes that are needed.
-3. Add support for the new syntax in `grammar/php.y`. Regenerate the parser by running
-   `php grammar/rebuildParsers.php`. Use `--debug` if there are conflicts.
-4. Add pretty-printing support by implementing a `pFooBar()` method in `PrettyPrinter\Standard`.
-5. Add tests both in `test/code/parser` and `test/code/prettyPrinter`.
-6. Add support for formatting-preserving pretty-printing. This is done by modifying the data tables
-   at the end of `PrettyPrinterAbstract`. Add a test in `test/code/formatPreservation`.
-7. Does the new syntax feature namespaced names? If so, add support for name resolution in
-   `NodeVisitor\NameResolver`. Test it in `NodeVisitor\NameResolverTest`.
-8. Does the new syntax require any changes to builders? Is so, make them :)

Makefile

@@ -1,13 +0,0 @@
-.PHONY: phpstan php-cs-fixer
-
-tools/vendor:
-	composer install -d tools
-
-phpstan: tools/vendor
-	php tools/vendor/bin/phpstan
-
-php-cs-fixer: tools/vendor
-	php tools/vendor/bin/php-cs-fixer fix
-
-tests:
-	php vendor/bin/phpunit

README.md

@@ -3,24 +3,24 @@ PHP Parser
 
 [![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
 
-This is a PHP parser written in PHP. Its purpose is to simplify static code analysis and
+This is a PHP 5.2 to PHP 8.2 parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
-[**Documentation for version 5.x**][doc_master] (current; for running on PHP >= 7.4; for parsing PHP 7.0 to PHP 8.4, with limited support for parsing PHP 5.x).
+[**Documentation for version 4.x**][doc_4_x] (stable; for running on PHP >= 7.1; for parsing PHP 5.2 to PHP 8.2).
 
-[Documentation for version 4.x][doc_4_x] (supported; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 8.3).
+[Documentation for version 3.x][doc_3_x] (unsupported; for running on PHP >= 5.5; for parsing PHP 5.2 to PHP 7.2).
 
 Features
 --------
 
 The main features provided by this library are:
 
- * Parsing PHP 7, and PHP 8 code into an abstract syntax tree (AST).
+ * Parsing PHP 5, PHP 7, and PHP 8 code into an abstract syntax tree (AST).
    * Invalid code can be parsed into a partial AST.
    * The AST contains accurate location information.
  * Dumping the AST in human-readable form.
  * Converting an AST back to PHP code.
-   * Formatting can be preserved for partially changed ASTs.
+   * Experimental: Formatting can be preserved for partially changed ASTs.
  * Infrastructure to traverse and modify ASTs.
  * Resolution of namespaced names.
  * Evaluation of constant expressions.
@@ -51,7 +51,7 @@ function test($foo)
 }
 CODE;
 
-$parser = (new ParserFactory())->createForNewestSupportedVersion();
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
 try {
     $ast = $parser->parse($code);
 } catch (Error $error) {
@@ -68,17 +68,12 @@ This dumps an AST looking something like this:
 ```
 array(
     0: Stmt_Function(
-        attrGroups: array(
-        )
         byRef: false
         name: Identifier(
             name: test
         )
         params: array(
             0: Param(
-                attrGroups: array(
-                )
-                flags: 0
                 type: null
                 byRef: false
                 variadic: false
@@ -93,11 +88,12 @@ array(
             0: Stmt_Expression(
                 expr: Expr_FuncCall(
                     name: Name(
-                        name: var_dump
+                        parts: array(
+                            0: var_dump
+                        )
                     )
                     args: array(
                         0: Arg(
-                            name: null
                             value: Expr_Variable(
                                 name: foo
                             )
@@ -139,16 +135,12 @@ This gives us an AST where the `Function_::$stmts` are empty:
 ```
 array(
     0: Stmt_Function(
-        attrGroups: array(
-        )
         byRef: false
         name: Identifier(
             name: test
         )
         params: array(
             0: Param(
-                attrGroups: array(
-                )
                 type: null
                 byRef: false
                 variadic: false
@@ -211,8 +203,9 @@ Component documentation:
  * [AST builders](doc/component/AST_builders.markdown)
    * Fluent builders for AST nodes
  * [Lexer](doc/component/Lexer.markdown)
-   * Emulation
+   * Lexer options
-   * Tokens, positions and attributes
+   * Token and file positions for nodes
+   * Custom attributes
  * [Error handling](doc/component/Error_handling.markdown)
    * Column information for errors
    * Error recovery (parsing of syntactically incorrect code)
@@ -230,4 +223,3 @@ Component documentation:
 
  [doc_3_x]: https://github.com/nikic/PHP-Parser/tree/3.x/doc
  [doc_4_x]: https://github.com/nikic/PHP-Parser/tree/4.x/doc
- [doc_master]: https://github.com/nikic/PHP-Parser/tree/master/doc

UPGRADE-5.0.md

@@ -1,534 +0,0 @@
-Upgrading from PHP-Parser 4.x to 5.0
-====================================
-
-### PHP version requirements
-
-PHP-Parser now requires PHP 7.4 or newer to run. It is however still possible to *parse* code for older versions, while running on a newer version.
-
-### PHP 5 parsing support
-
-The dedicated parser for PHP 5 has been removed. The PHP 7 parser now accepts a `PhpVersion` argument, which can be used to improve compatibility with older PHP versions.
-
-In particular, if an older `PhpVersion` is specified, then:
-
- * For versions before PHP 7.0, `$foo =& new Bar()` assignments are allowed without error.
- * For versions before PHP 7.0, invalid octal literals `089` are allowed without error.
- * For versions before PHP 7.0, unicode escape sequences `\u{123}` in strings are not parsed.
- * Type hints are interpreted as a class `Name` or as a built-in `Identifier` depending on PHP
-   version, for example `int` is treated as a class name on PHP 5.6 and as a built-in on PHP 7.0.
-
-However, some aspects of PHP 5 parsing are no longer supported:
-
- * Some variables like `$$foo[0]` are valid in both PHP 5 and PHP 7, but have different interpretation. In that case, the PHP 7 AST will always be constructed (`($$foo)[0]` rather than `${$foo[0]}`).
- * Declarations of the form `global $$var[0]` are not supported in PHP 7 and will cause a parse error. In error recovery mode, it is possible to continue parsing after such declarations.
- * The PHP 7 parser will accept many constructs that are not valid in PHP 5. However, this was also true of the dedicated PHP 5 parser.
-
-The following symbols are affected by this removal:
-
- * The `PhpParser\Parser\Php5` class has been removed.
- * The `PhpParser\Parser\Multiple` class has been removed. While not strictly related to PHP 5 support, this functionality is no longer useful without it.
- * The `PhpParser\ParserFactory::ONLY_PHP5` and `PREFER_PHP5` options have been removed.
-
-### Changes to the parser factory
-
-The `ParserFactory::create()` method has been removed in favor of three new methods that provide more fine-grained control over the PHP version being targeted:
-
- * `createForNewestSupportedVersion()`: Use this if you don't know the PHP version of the code you're parsing. It's better to assume a too new version than a too old one.
- * `createForHostVersion()`: Use this if you're parsing code for the PHP version you're running on.
- * `createForVersion()`: Use this if you know the PHP version of the code you want to parse.
-
-The `createForNewestSupportedVersion()` and `createForHostVersion()` are available since PHP-Parser 4.18.0, to allow libraries to support PHP-Parser 4 and 5 at the same time more easily.
-
-In all cases, the PHP version is a fairly weak hint that is only used on a best-effort basis. The parser will usually accept code for newer versions if it does not have any backwards-compatibility implications.
-
-For example, if you specify version `"8.0"`, then `class ReadOnly {}` is treated as a valid class declaration, while using `public readonly int $prop` will lead to a parse error. However, `final public const X = Y;` will be accepted in both cases.
-
-```php
-use PhpParser\ParserFactory;
-use PhpParser\PhpVersion;
-
-$factory = new ParserFactory();
-
-# Before
-$parser = $factory->create(ParserFactory::PREFER_PHP7);
-
-# After (this is roughly equivalent to PREFER_PHP7 behavior)
-$parser = $factory->createForNewestSupportedVersion();
-# Or
-$parser = $factory->createForHostVersion();
-
-# Before
-$parser = $factory->create(ParserFactory::ONLY_PHP5);
-# After (supported on a best-effort basis)
-$parser = $factory->createForVersion(PhpVersion::fromString("5.6"));
-```
-
-### Changes to the throw representation
-
-Previously, `throw` statements like `throw $e;` were represented using the `Stmt\Throw_` class,
-while uses inside other expressions (such as `$x ?? throw $e`) used the `Expr\Throw_` class.
-
-Now, `throw $e;` is represented as a `Stmt\Expression` that contains an `Expr\Throw_`. The
-`Stmt\Throw_` class has been removed.
-
-```php
-# Code
-throw $e;
-
-# Before
-Stmt_Throw(
-    expr: Expr_Variable(
-        name: e
-    )
-)
-
-# After
-Stmt_Expression(
-    expr: Expr_Throw(
-        expr: Expr_Variable(
-            name: e
-        )
-    )
-)
-```
-
-### Changes to the array destructuring representation
-
-Previously, the `list($x) = $y` destructuring syntax was represented using a `Node\Expr\List_`
-node, while `[$x] = $y` used a `Node\Expr\Array_` node, the same used for the creation (rather than
-destructuring) of arrays.
-
-Now, destructuring is always represented using `Node\Expr\List_`. The `kind` attribute with value
-`Node\Expr\List_::KIND_LIST` or `Node\Expr\List_::KIND_ARRAY` specifies which syntax was actually
-used.
-
-```php
-# Code
-[$x] = $y;
-
-# Before
-Expr_Assign(
-   var: Expr_Array(
-       items: array(
-           0: Expr_ArrayItem(
-               key: null
-               value: Expr_Variable(
-                   name: x
-               )
-               byRef: false
-               unpack: false
-           )
-       )
-   )
-   expr: Expr_Variable(
-       name: y
-   )
-)
-
-# After
-Expr_Assign(
-   var: Expr_List(
-       items: array(
-           0: ArrayItem(
-               key: null
-               value: Expr_Variable(
-                   name: x
-               )
-               byRef: false
-               unpack: false
-           )
-       )
-   )
-   expr: Expr_Variable(
-       name: y
-   )
-)
-```
-
-### Changes to the name representation
-
-Previously, `Name` nodes had a `parts` subnode, which stores an array of name parts, split by
-namespace separators. Now, `Name` nodes instead have a `name` subnode, which stores a plain string.
-
-For example, the name `Foo\Bar` was previously represented by `Name(parts: ['Foo', 'Bar'])` and is
-now represented by `Name(name: 'Foo\Bar')` instead.
-
-It is possible to convert the name to the previous representation using `$name->getParts()`. The
-`Name` constructor continues to accept both the string and the array representation.
-
-The `Name::getParts()` method is available since PHP-Parser 4.16.0, to allow libraries to support
-PHP-Parser 4 and 5 at the same time more easily.
-
-### Changes to the block representation
-
-Previously, code blocks `{ ... }` were always flattened into their parent statement list. For
-example `while ($x) { $a; { $b; } $c; }` would produce the same node structure as
-`if ($x) { $a; $b; $c; }`, namely a `Stmt\While_` node whose `stmts` subnode is an array of three
-statements.
-
-Now, the nested `{ $b; }` block is represented using an explicit `Stmt\Block` node. However, the
-outer `{ $a; { $b; } $c; }` block is still represented using a simple array in the `stmts` subnode.
-
-```php
-# Code
-while ($x) { $a; { $b; } $c; }
-
-# Before
-Stmt_While(
-    cond: Expr_Variable(
-        name: x
-    )
-    stmts: array(
-        0: Stmt_Expression(
-            expr: Expr_Variable(
-                name: a
-            )
-        )
-        1: Stmt_Expression(
-            expr: Expr_Variable(
-                name: b
-            )
-        )
-        2: Stmt_Expression(
-            expr: Expr_Variable(
-                name: c
-            )
-        )
-    )
-)
-
-# After
-Stmt_While(
-    cond: Expr_Variable(
-        name: x
-    )
-    stmts: array(
-        0: Stmt_Expression(
-            expr: Expr_Variable(
-                name: a
-            )
-        )
-        1: Stmt_Block(
-            stmts: array(
-                0: Stmt_Expression(
-                    expr: Expr_Variable(
-                        name: b
-                    )
-                )
-            )
-        )
-        2: Stmt_Expression(
-            expr: Expr_Variable(
-                name: c
-            )
-        )
-    )
-)
-```
-
-### Changes to comment assignment
-
-Previously, comments were assigned to all nodes starting at the same position. Now they will be
-assigned to the outermost node only.
-
-```php
-# Code
-// Comment
-$a + $b;
-
-# Before
-Stmt_Expression(
-    expr: Expr_BinaryOp_Plus(
-        left: Expr_Variable(
-            name: a
-            comments: array(
-                0: // Comment
-            )
-        )
-        right: Expr_Variable(
-            name: b
-        )
-        comments: array(
-            0: // Comment
-        )
-    )
-    comments: array(
-        0: // Comment
-    )
-)
-
-# After
-Stmt_Expression(
-    expr: Expr_BinaryOp_Plus(
-        left: Expr_Variable(
-            name: a
-        )
-        right: Expr_Variable(
-            name: b
-        )
-    )
-    comments: array(
-        0: // Comment
-    )
-)
-```
-
-### Renamed nodes
-
-A number of AST nodes have been renamed or moved in the AST hierarchy:
-
- * `Node\Scalar\LNumber` is now `Node\Scalar\Int_`.
- * `Node\Scalar\DNumber` is now `Node\Scalar\Float_`.
- * `Node\Scalar\Encapsed` is now `Node\Scalar\InterpolatedString`.
- * `Node\Scalar\EncapsedStringPart` is now `Node\InterpolatedStringPart` and no longer extends
-   `Node\Scalar` or `Node\Expr`.
- * `Node\Expr\ArrayItem` is now `Node\ArrayItem` and no longer extends `Node\Expr`.
- * `Node\Expr\ClosureUse` is now `Node\ClosureUse` and no longer extends `Node\Expr`.
- * `Node\Stmt\DeclareDeclare` is now `Node\DeclareItem` and no longer extends `Node\Stmt`.
- * `Node\Stmt\PropertyProperty` is now `Node\PropertyItem` and no longer extends `Node\Stmt`.
- * `Node\Stmt\StaticVar` is now `Node\StaticVar` and no longer extends `Node\Stmt`.
- * `Node\Stmt\UseUse` is now `Node\UseItem` and no longer extends `Node\Stmt`.
-
-The old class names have been retained as aliases for backwards compatibility. However, the `Node::getType()` method will now always return the new name (e.g. `ClosureUse` instead of `Expr_ClosureUse`).
-
-### Modifiers
-
-Modifier flags (as used by the `$flags` subnode of `Class_`, `ClassMethod`, `Property`, etc.) are now available as class constants on a separate `PhpParser\Modifiers` class, instead of being part of `PhpParser\Node\Stmt\Class_`, to make it clearer that these are used by many different nodes. The old constants are deprecated, but are still available.
-
-```php
-PhpParser\Node\Stmt\Class_::MODIFIER_PUBLIC    -> PhpParser\Modifiers::PUBLIC
-PhpParser\Node\Stmt\Class_::MODIFIER_PROTECTED -> PhpParser\Modifiers::PROTECTED
-PhpParser\Node\Stmt\Class_::MODIFIER_PRIVATE   -> PhpParser\Modifiers::PRIVATE
-PhpParser\Node\Stmt\Class_::MODIFIER_STATIC    -> PhpParser\Modifiers::STATIC
-PhpParser\Node\Stmt\Class_::MODIFIER_ABSTRACT  -> PhpParser\Modifiers::ABSTRACT
-PhpParser\Node\Stmt\Class_::MODIFIER_FINAL     -> PhpParser\Modifiers::FINAL
-PhpParser\Node\Stmt\Class_::MODIFIER_READONLY  -> PhpParser\Modifiers::READONLY
-PhpParser\Node\Stmt\Class_::VISIBILITY_MODIFIER_MASK -> PhpParser\Modifiers::VISIBILITY_MASK
-```
-
-### Changes to node constructors
-
-Node constructor arguments accepting types no longer accept plain strings. Either an `Identifier` or `Name` (or `ComplexType`) should be passed instead. This affects the following constructor arguments:
-
-* The `'returnType'` key of `$subNodes` argument of `Node\Expr\ArrowFunction`.
-* The `'returnType'` key of `$subNodes` argument of `Node\Expr\Closure`.
-* The `'returnType'` key of `$subNodes` argument of `Node\Stmt\ClassMethod`.
-* The `'returnType'` key of `$subNodes` argument of `Node\Stmt\Function_`.
-* The `$type` argument of `Node\NullableType`.
-* The `$type` argument of `Node\Param`.
-* The `$type` argument of `Node\Stmt\Property`.
-* The `$type` argument of `Node\ClassConst`.
-
-To follow the previous behavior, an `Identifier` should be passed, which indicates a built-in type.
-
-### Changes to the pretty printer
-
-A number of changes to the standard pretty printer have been made, to make it match contemporary coding style conventions (and in particular PSR-12). Options to restore the previous behavior are not provided, but it is possible to override the formatting methods (such as `pStmt_ClassMethod`) with your preferred formatting.
-
-Return types are now formatted without a space before the `:`:
-
-```php
-# Before
-function test() : Type
-{
-}
-
-# After
-function test(): Type
-{
-}
-```
-
-`abstract` and `final` are now printed before visibility modifiers:
-
-```php
-# Before
-public abstract function test();
-
-# After
-abstract public function test();
-```
-
-A space is now printed between `use` and the following `(` for closures:
-
-```php
-# Before
-function () use($var) {
-};
-
-# After
-function () use ($var) {
-};
-```
-
-Backslashes in single-quoted strings are now only printed if they are necessary:
-
-```php
-# Before
-'Foo\\Bar';
-'\\\\';
-
-# After
-'Foo\Bar';
-'\\\\';
-```
-
-`else if` structures will now omit redundant parentheses:
-
-```php
-# Before
-else {
-    if ($x) {
-        // ...
-    }
-}
-
-# After
-else if ($x) {
-     // ...
-}
-```
-
-The pretty printer now accepts a `phpVersion` option, which accepts a `PhpVersion` object and defaults to PHP 7.4. The pretty printer will make formatting choices to make the code valid for that version. It currently controls the following behavior:
-
-* For PHP >= 7.0 (default), short array syntax `[]` will be used by default. This does not affect nodes that specify an explicit array syntax using the `kind` attribute.
-* For PHP >= 7.0 (default), parentheses around `yield` expressions will only be printed when necessary. Previously, parentheses were always printed, even if `yield` was used as a statement.
-* For PHP >= 7.1 (default), the short array syntax `[]` will be used for destructuring by default (instead of `list()`). This does not affect nodes that specify an explicit syntax using the `kind` attribute.
-* For PHP >= 7.3 (default), a newline is no longer forced after heredoc/nowdoc strings, as the requirement for this has been removed with the introduction of flexible heredoc/nowdoc strings.
-* For PHP >= 7.3 (default), heredoc/nowdoc strings are now indented just like regular code. This was allowed with the introduction of flexible heredoc/nowdoc strings.
-
-### Changes to precedence handling in the pretty printer
-
-The pretty printer now more accurately models operator precedence. Especially for unary operators, less unnecessary parentheses will be printed. Conversely, many bugs where semantically meaningful parentheses were omitted have been fixed.
-
-To support these changes, precedence is now handled differently in the pretty printer. The internal `p()` method, which is used to recursively print nodes, now has the following signature:
-```php
-protected function p(
-    Node $node, int $precedence = self::MAX_PRECEDENCE, int $lhsPrecedence = self::MAX_PRECEDENCE,
-    bool $parentFormatPreserved = false
-): string;
-```
-
-The `$precedence` is the precedence of the direct parent operator (if any), while `$lhsPrecedence` is that precedence of the nearest binary operator on whose left-hand-side the node occurs. For unary operators, only the `$lhsPrecedence` is relevant.
-
-Recursive calls in pretty-printer methods should generally continue calling `p()` without additional parameters. However, pretty-printer methods for operators that participate in precedence resolution need to be adjusted. For example, typical implementations for operators look as follows now:
-
-```php
-protected function pExpr_BinaryOp_Plus(
-    BinaryOp\Plus $node, int $precedence, int $lhsPrecedence
-): string {
-    return $this->pInfixOp(
-        BinaryOp\Plus::class, $node->left, ' + ', $node->right, $precedence, $lhsPrecedence);
-}
-
-protected function pExpr_UnaryPlus(
-    Expr\UnaryPlus $node, int $precedence, int $lhsPrecedence
-): string {
-    return $this->pPrefixOp(Expr\UnaryPlus::class, '+', $node->expr, $precedence, $lhsPrecedence);
-}
-```
-
-The new `$precedence` and `$lhsPrecedence` arguments need to be passed down to the `pInfixOp()`, `pPrefixOp()` and `pPostfixOp()` methods.
-
-### Changes to the node traverser
-
-If there are multiple visitors, the node traverser will now call `leaveNode()` and `afterTraverse()` methods in the reverse order of the corresponding `enterNode()` and `beforeTraverse()` calls:
-
-```php
-# Before
-$visitor1->enterNode($node);
-$visitor2->enterNode($node);
-$visitor1->leaveNode($node);
-$visitor2->leaveNode($node);
-
-# After
-$visitor1->enterNode($node);
-$visitor2->enterNode($node);
-$visitor2->leaveNode($node);
-$visitor1->leaveNode($node);
-```
-
-Additionally, the special `NodeVisitor` return values have been moved from `NodeTraverser` to `NodeVisitor`. The old names are deprecated, but still available.
-
-```php
-PhpParser\NodeTraverser::REMOVE_NODE -> PhpParser\NodeVisitor::REMOVE_NODE
-PhpParser\NodeTraverser::DONT_TRAVERSE_CHILDREN -> PhpParser\NodeVisitor::DONT_TRAVERSE_CHILDREN
-PhpParser\NodeTraverser::DONT_TRAVERSE_CURRENT_AND_CHILDREN -> PhpParser\NodeVisitor::DONT_TRAVERSE_CURRENT_AND_CHILDREN
-PhpParser\NodeTraverser::STOP_TRAVERSAL -> PhpParser\NodeVisitor::STOP_TRAVERSAL
-```
-
-Visitors can now also be passed directly to the `NodeTraverser` constructor:
-
-```php
-# Before (and still supported)
-$traverser = new NodeTraverser();
-$traverser->addVisitor(new NameResolver());
-
-# After
-$traverser = new NodeTraverser(new NameResolver());
-```
-
-### Changes to token representation
-
-Tokens are now internally represented using the `PhpParser\Token` class, which exposes the same base interface as
-the `PhpToken` class introduced in PHP 8.0. On PHP 8.0 or newer, `PhpParser\Token` extends from `PhpToken`, otherwise
-it extends from a polyfill implementation. The most important parts of the interface may be summarized as follows:
-
-```php
-class Token {
-    public int $id;
-    public string $text;
-    public int $line;
-    public int $pos;
-
-    public function is(int|string|array $kind): bool;
-}
-```
-
-The token array is now an array of `Token`s, rather than an array of arrays and strings.
-Additionally, the token array is now terminated by a sentinel token with ID 0.
-
-### Changes to the lexer
-
-The lexer API is reduced to a single `Lexer::tokenize()` method, which returns an array of tokens. The `startLexing()` and `getNextToken()` methods have been removed.
-
-Responsibility for determining start and end attributes for nodes has been moved from the lexer to the parser. The lexer no longer accepts an options array. The `usedAttributes` option has been removed without replacement, and the parser will now unconditionally add the `comments`, `startLine`, `endLine`, `startFilePos`, `endFilePos`, `startTokenPos` and `endTokenPos` attributes.
-
-There should no longer be a need to directly interact with the `Lexer` for end users, as the `ParserFactory` will create an appropriate instance, and no additional configuration of the lexer is necessary. To use formatting-preserving pretty printing, the setup boilerplate changes as follows:
-
-```php
-# Before
-
-$lexer = new Lexer\Emulative([
-    'usedAttributes' => [
-        'comments',
-        'startLine', 'endLine',
-        'startTokenPos', 'endTokenPos',
-    ],
-]);
-
-$parser = new Parser\Php7($lexer);
-$oldStmts = $parser->parse($code);
-$oldTokens = $lexer->getTokens();
-
-$traverser = new NodeTraverser();
-$traverser->addVisitor(new NodeVisitor\CloningVisitor());
-$newStmts = $traverser->traverse($oldStmts);
-
-# After
-
-$parser = (new ParserFactory())->createForNewestSupportedVersion();
-$oldStmts = $parser->parse($code);
-$oldTokens = $parser->getTokens();
-
-$traverser = new NodeTraverser(new NodeVisitor\CloningVisitor());
-$newStmts = $traverser->traverse($oldStmts);
-```
-
-### Miscellaneous changes
-
- * The deprecated `Builder\Param::setTypeHint()` method has been removed in favor of `Builder\Param::setType()`.
- * The deprecated `Error` constructor taking a start line has been removed. Pass `['startLine' => $startLine]` attributes instead.
- * The deprecated `Comment::getLine()`, `Comment::getTokenPos()` and `Comment::getFilePos()` methods have been removed. Use `Comment::getStartLine()`, `Comment::getStartTokenPos()` and `Comment::getStartFilePos()` instead.
- * `Comment::getReformattedText()` now normalizes CRLF newlines to LF newlines.
- * The `Node::getLine()` method has been deprecated. Use `Node::getStartLine()` instead.

bin/php-parse

@@ -26,7 +26,13 @@ if (empty($files)) {
     showHelp("Must specify at least one file.");
 }
 
-$parser = (new PhpParser\ParserFactory())->createForVersion($attributes['version']);
+$lexer = new PhpParser\Lexer\Emulative(['usedAttributes' => [
+    'startLine', 'endLine', 'startFilePos', 'endFilePos', 'comments'
+]]);
+$parser = (new PhpParser\ParserFactory)->create(
+    PhpParser\ParserFactory::PREFER_PHP7,
+    $lexer
+);
 $dumper = new PhpParser\NodeDumper([
     'dumpComments' => true,
     'dumpPositions' => $attributes['with-positions'],
@@ -37,10 +43,7 @@ $traverser = new PhpParser\NodeTraverser();
 $traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);
 
 foreach ($files as $file) {
-    if ($file === '-') {
+    if (strpos($file, '<?php') === 0) {
-        $code = file_get_contents('php://stdin');
-        fwrite(STDERR, "====> Stdin:\n");
-    } else if (strpos($file, '<?php') === 0) {
         $code = $file;
         fwrite(STDERR, "====> Code $code\n");
     } else {
@@ -105,7 +108,7 @@ function showHelp($error = '') {
     if ($error) {
         fwrite(STDERR, $error . "\n\n");
     }
-    fwrite($error ? STDERR : STDOUT, <<<'OUTPUT'
+    fwrite($error ? STDERR : STDOUT, <<<OUTPUT
 Usage: php-parse [operations] file1.php [file2.php ...]
    or: php-parse [operations] "<?php code"
 Turn PHP source code into an abstract syntax tree.
@@ -120,7 +123,6 @@ Operations is a list of the following options (--dump by default):
     -c, --with-column-info  Show column-numbers for errors (if available)
     -P, --with-positions    Show positions in node dumps
     -r, --with-recovery     Use parsing with error recovery
-        --version=VERSION   Target specific PHP version (default: newest)
     -h, --help              Display this page
 
 Example:
@@ -141,7 +143,6 @@ function parseArgs($args) {
         'with-column-info' => false,
         'with-positions' => false,
         'with-recovery' => false,
-        'version' => PhpParser\PhpVersion::getNewestSupported(),
     ];
 
     array_shift($args);
@@ -192,9 +193,7 @@ function parseArgs($args) {
                 $parseOptions = false;
                 break;
             default:
-                if (preg_match('/^--version=(.*)$/', $arg, $matches)) {
+                if ($arg[0] === '-') {
-                    $attributes['version'] = PhpParser\PhpVersion::fromString($matches[1]);
-                } elseif ($arg[0] === '-' && \strlen($arg[0]) > 1) {
                     showHelp("Invalid operation $arg.");
                 } else {
                     $files[] = $arg;

composer.json

@@ -13,18 +13,16 @@
         }
     ],
     "require": {
-        "php": ">=7.4",
+        "php": ">=7.1",
-        "ext-tokenizer": "*",
+        "ext-tokenizer": "*"
-        "ext-json": "*",
-        "ext-ctype": "*"
     },
     "require-dev": {
-        "phpunit/phpunit": "^9.0",
+        "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0",
         "ircmaxell/php-yacc": "^0.0.7"
     },
     "extra": {
         "branch-alias": {
-            "dev-master": "5.0-dev"
+            "dev-master": "4.9-dev"
         }
     },
     "autoload": {

doc/0_Introduction.markdown

@@ -1,7 +1,7 @@
 Introduction
 ============
 
-This project is a PHP parser **written in PHP itself**.
+This project is a PHP 5.2 to PHP 8.0 parser **written in PHP itself**.
 
 What is this for?
 -----------------
@@ -13,42 +13,29 @@ application dealing with code programmatically. A parser constructs an [Abstract
 There are other ways of processing source code. One that PHP supports natively is using the
 token stream generated by [`token_get_all`][2]. The token stream is much more low level than
 the AST and thus has different applications: It allows to also analyze the exact formatting of
-a file. On the other hand, the token stream is much harder to deal with for more complex analysis.
+a file. On the other hand the token stream is much harder to deal with for more complex analysis.
 For example, an AST abstracts away the fact that, in PHP, variables can be written as `$foo`, but also
 as `$$bar`, `${'foobar'}` or even `${!${''}=barfoo()}`. You don't have to worry about recognizing
 all the different syntaxes from a stream of tokens.
 
 Another question is: Why would I want to have a PHP parser *written in PHP*? Well, PHP might not be
 a language especially suited for fast parsing, but processing the AST is much easier in PHP than it
-would be in other, faster languages like C. Furthermore the people most likely wanting to do
+would be in other, faster languages like C. Furthermore the people most probably wanting to do
 programmatic PHP code analysis are incidentally PHP developers, not C developers.
 
 What can it parse?
 ------------------
 
-The parser supports parsing PHP 7 and PHP 8 code, with the following exceptions:
+The parser supports parsing PHP 5.2-8.0, with the following exceptions:
 
  * Namespaced names containing whitespace (e.g. `Foo \ Bar` instead of `Foo\Bar`) are not supported.
-   These are illegal in PHP 8, but are legal in earlier versions. However, PHP-Parser does not
+   These are illegal in PHP 8, but are legal in earlier version. However, PHP-Parser does not
    support them for any version.
 
-PHP-Parser 4.x had full support for parsing PHP 5. PHP-Parser 5.x has only limited support, with the
-following caveats:
-
- * Some variable expressions like `$$foo[0]` are valid in both PHP 5 and PHP 7, but have different 
-   interpretation. In such cases, the PHP 7 AST will always be constructed (using `($$foo)[0]`
-   rather than `${$foo[0]}`).
- * Declarations of the form `global $$var[0]` are not supported in PHP 7 and will cause a parse 
-   error. In error recovery mode, it is possible to continue parsing after such declarations.
-
 As the parser is based on the tokens returned by `token_get_all` (which is only able to lex the PHP
 version it runs on), additionally a wrapper for emulating tokens from newer versions is provided.
-This allows to parse PHP 8.4 source code running on PHP 7.4, for example. This emulation is not
+This allows to parse PHP 7.4 source code running on PHP 7.1, for example. This emulation is somewhat
-perfect, but works well in practice.
+hacky and not perfect, but it should work well on any sane code.
-
-Finally, it should be noted that the parser aims to accept all valid code, not reject all invalid
-code. It will generally accept code that is only valid in newer versions (even when targeting an
-older one), and accept code that is syntactically correct, but would result in a compiler error.
 
 What output does it produce?
 ----------------------------
@@ -76,21 +63,21 @@ This matches the structure of the code: An echo statement, which takes two strin
 with the values `Hi` and `World`.
 
 You can also see that the AST does not contain any whitespace information (but most comments are saved).
-However, it does retain accurate position information, which can be used to inspect precise formatting.
+So using it for formatting analysis is not possible.
 
 What else can it do?
 --------------------
 
-Apart from the parser itself, this package also bundles support for some other, related features:
+Apart from the parser itself this package also bundles support for some other, related features:
 
  * Support for pretty printing, which is the act of converting an AST into PHP code. Please note
    that "pretty printing" does not imply that the output is especially pretty. It's just how it's
    called ;)
- * Support for serializing and unserializing the node tree to JSON.
+ * Support for serializing and unserializing the node tree to JSON
- * Support for dumping the node tree in a human-readable form (see the section above for an
+ * Support for dumping the node tree in a human readable form (see the section above for an
-   example of how the output looks like).
+   example of how the output looks like)
- * Infrastructure for traversing and changing the AST (node traverser and node visitors).
+ * Infrastructure for traversing and changing the AST (node traverser and node visitors)
- * A node visitor for resolving namespaced names.
+ * A node visitor for resolving namespaced names
 
  [0]: http://en.wikipedia.org/wiki/Static_program_analysis
  [1]: http://en.wikipedia.org/wiki/Abstract_syntax_tree

doc/2_Usage_of_basic_components.markdown

@@ -12,7 +12,7 @@ To bootstrap the library, include the autoloader generated by composer:
 require 'path/to/vendor/autoload.php';
 ```
 
-Additionally, you may want to set the `xdebug.max_nesting_level` ini option to a higher value:
+Additionally you may want to set the `xdebug.max_nesting_level` ini option to a higher value:
 
 ```php
 ini_set('xdebug.max_nesting_level', 3000);
@@ -29,29 +29,25 @@ In order to parse code, you first have to create a parser instance:
 
 ```php
 use PhpParser\ParserFactory;
-use PhpParser\PhpVersion;
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
-
-// Parser for the version you are running on.
-$parser = (new ParserFactory())->createForHostVersion();
-
-// Parser for the newest PHP version supported by the PHP-Parser library.
-$parser = (new ParserFactory())->createForNewestSupportedVersion();
-
-// Parser for a specific PHP version.
-$parser = (new ParserFactory())->createForVersion(PhpVersion::fromString('8.1'));
 ```
 
-Which version you should target depends on your use case. In many cases you will want to use the
+The factory accepts a kind argument, that determines how different PHP versions are treated:
-host version, as people typically analyze code for the version they are running on. However, when
-analyzing arbitrary code you are usually best off using the newest supported version, which tends
-to accept the widest range of code (unless there are breaking changes in PHP).
 
-The `createXYZ()` methods optionally accept an array of lexer options. Some use cases that require
+Kind | Behavior
-customized lexer options are discussed in the [lexer documentation](component/Lexer.markdown).
+-----|---------
+`ParserFactory::PREFER_PHP7` | Try to parse code as PHP 7. If this fails, try to parse it as PHP 5.
+`ParserFactory::PREFER_PHP5` | Try to parse code as PHP 5. If this fails, try to parse it as PHP 7.
+`ParserFactory::ONLY_PHP7` | Parse code as PHP 7.
+`ParserFactory::ONLY_PHP5` | Parse code as PHP 5.
 
-Subsequently, you can pass PHP code (including the opening `<?php` tag) to the `parse()` method in
+Unless you have a strong reason to use something else, `PREFER_PHP7` is a reasonable default.
-order to create a syntax tree. If a syntax error is encountered, a `PhpParser\Error` exception will
+
-be thrown by default:
+The `create()` method optionally accepts a `Lexer` instance as the second argument. Some use cases
+that require customized lexers are discussed in the [lexer documentation](component/Lexer.markdown).
+
+Subsequently you can pass PHP code (including the opening `<?php` tag) to the `parse` method in order to
+create a syntax tree. If a syntax error is encountered, an `PhpParser\Error` exception will be thrown:
 
 ```php
 <?php
@@ -66,13 +62,13 @@ function printLine($msg) {
 printLine('Hello World!!!');
 CODE;
 
-$parser = (new ParserFactory())->createForHostVersion();
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
 
 try {
     $stmts = $parser->parse($code);
     // $stmts is an array of statement nodes
 } catch (Error $e) {
-    echo 'Parse Error: ', $e->getMessage(), "\n";
+    echo 'Parse Error: ', $e->getMessage();
 }
 ```
 
@@ -81,7 +77,7 @@ A parser instance can be reused to parse multiple files.
 Node dumping
 ------------
 
-To dump the abstract syntax tree in human-readable form, a `NodeDumper` can be used:
+To dump the abstract syntax tree in human readable form, a `NodeDumper` can be used:
 
 ```php
 <?php
@@ -96,17 +92,12 @@ For the sample code from the previous section, this will produce the following o
 ```
 array(
     0: Stmt_Function(
-        attrGroups: array(
-        )
         byRef: false
         name: Identifier(
             name: printLine
         )
         params: array(
             0: Param(
-                attrGroups: array(
-                )
-                flags: 0
                 type: null
                 byRef: false
                 variadic: false
@@ -134,11 +125,12 @@ array(
     1: Stmt_Expression(
         expr: Expr_FuncCall(
             name: Name(
-                name: printLine
+                parts: array(
+                    0: printLine
+                )
             )
             args: array(
                 0: Arg(
-                    name: null
                     value: Scalar_String(
                         value: Hello World!!!
                     )
@@ -179,7 +171,7 @@ with them easier they are grouped into three categories:
 
  * `PhpParser\Node\Stmt`s are statement nodes, i.e. language constructs that do not return
    a value and can not occur in an expression. For example a class definition is a statement.
-   It doesn't return a value, and you can't write something like `func(class A {});`.
+   It doesn't return a value and you can't write something like `func(class A {});`.
  * `PhpParser\Node\Expr`s are expression nodes, i.e. language constructs that return a value
    and thus can occur in other expressions. Examples of expressions are `$var`
    (`PhpParser\Node\Expr\Variable`) and `func()` (`PhpParser\Node\Expr\FuncCall`).
@@ -206,19 +198,18 @@ without the `PhpParser\Node\` prefix and `\` replaced with `_`. It also does not
 It is possible to associate custom metadata with a node using the `setAttribute()` method. This data
 can then be retrieved using `hasAttribute()`, `getAttribute()` and `getAttributes()`.
 
-By default, the parser adds the `startLine`, `endLine`, `startTokenPos`, `endTokenPos`,
+By default the lexer adds the `startLine`, `endLine` and `comments` attributes. `comments` is an array
-`startFilePos`, `endFilePos` and `comments` attributes. `comments` is an array of
+of `PhpParser\Comment[\Doc]` instances.
-`PhpParser\Comment[\Doc]` instances.
 
-The pre-defined attributes can also be accessed using `getStartLine()` instead of
+The start line can also be accessed using `getLine()`/`setLine()` (instead of `getAttribute('startLine')`).
-`getAttribute('startLine')`, and so on. The last doc comment from the `comments` attribute can be
+The last doc comment from the `comments` attribute can be obtained using `getDocComment()`.
-obtained using `getDocComment()`.
 
 Pretty printer
 --------------
 
-The pretty printer component compiles the AST back to PHP code according to a specified scheme.
+The pretty printer component compiles the AST back to PHP code. As the parser does not retain formatting
-Currently, there is only one scheme available, namely `PhpParser\PrettyPrinter\Standard`.
+information the formatting is done using a specified scheme. Currently there is only one scheme available,
+namely `PhpParser\PrettyPrinter\Standard`.
 
 ```php
 use PhpParser\Error;
@@ -227,8 +218,8 @@ use PhpParser\PrettyPrinter;
 
 $code = "<?php echo 'Hi ', hi\\getTarget();";
 
-$parser = (new ParserFactory())->createForHostVersion();
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
-$prettyPrinter = new PrettyPrinter\Standard();
+$prettyPrinter = new PrettyPrinter\Standard;
 
 try {
     // parse
@@ -246,7 +237,7 @@ try {
 
     echo $code;
 } catch (Error $e) {
-    echo 'Parse Error: ', $e->getMessage(), "\n";
+    echo 'Parse Error: ', $e->getMessage();
 }
 ```
 
@@ -254,7 +245,7 @@ The above code will output:
 
     echo 'Hello ', hi\getTarget();
 
-As you can see, the source code was first parsed using `PhpParser\Parser->parse()`, then changed and then
+As you can see the source code was first parsed using `PhpParser\Parser->parse()`, then changed and then
 again converted to code using `PhpParser\PrettyPrinter\Standard->prettyPrint()`.
 
 The `prettyPrint()` method pretty prints a statements array. It is also possible to pretty print only a
@@ -263,13 +254,10 @@ single expression using `prettyPrintExpr()`.
 The `prettyPrintFile()` method can be used to print an entire file. This will include the opening `<?php` tag
 and handle inline HTML as the first/last statement more gracefully.
 
-There is also a pretty-printing mode which retains formatting for parts of the AST that have not
-been changed, which requires additional setup.
-
 > Read more: [Pretty printing documentation](component/Pretty_printing.markdown)
 
-Node traversal
+Node traversation
---------------
+-----------------
 
 The above pretty printing example used the fact that the source code was known and thus it was easy to
 write code that accesses a certain part of a node tree and changes it. Normally this is not the case.
@@ -284,7 +272,7 @@ use PhpParser\NodeTraverser;
 use PhpParser\ParserFactory;
 use PhpParser\PrettyPrinter;
 
-$parser        = (new ParserFactory())->createForHostVersion();
+$parser        = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
 $traverser     = new NodeTraverser;
 $prettyPrinter = new PrettyPrinter\Standard;
 
@@ -315,7 +303,8 @@ The corresponding node visitor might look like this:
 use PhpParser\Node;
 use PhpParser\NodeVisitorAbstract;
 
-class MyNodeVisitor extends NodeVisitorAbstract {
+class MyNodeVisitor extends NodeVisitorAbstract
+{
     public function leaveNode(Node $node) {
         if ($node instanceof Node\Scalar\String_) {
             $node->value = 'foo';
@@ -337,7 +326,7 @@ public function afterTraverse(array $nodes);
 ```
 
 The `beforeTraverse()` method is called once before the traversal begins and is passed the nodes the
-traverser was called with. This method can be used for resetting values before traversal or
+traverser was called with. This method can be used for resetting values before traversation or
 preparing the tree for traversal.
 
 The `afterTraverse()` method is similar to the `beforeTraverse()` method, with the only difference that
@@ -349,18 +338,15 @@ i.e. before its subnodes are traversed, the latter when it is left.
 All four methods can either return the changed node or not return at all (i.e. `null`) in which
 case the current node is not changed.
 
-The `enterNode()` method can additionally return the value `NodeVisitor::DONT_TRAVERSE_CHILDREN`,
+The `enterNode()` method can additionally return the value `NodeTraverser::DONT_TRAVERSE_CHILDREN`,
 which instructs the traverser to skip all children of the current node. To furthermore prevent subsequent
-visitors from visiting the current node, `NodeVisitor::DONT_TRAVERSE_CURRENT_AND_CHILDREN` can be used instead.
+visitors from visiting the current node, `NodeTraverser::DONT_TRAVERSE_CURRENT_AND_CHILDREN` can be used instead.
 
-Both methods can additionally return the following values:
+The `leaveNode()` method can additionally return the value `NodeTraverser::REMOVE_NODE`, in which
-
+case the current node will be removed from the parent array. Furthermore it is possible to return
- * `NodeVisitor::STOP_TRAVERSAL`, in which case no further nodes will be visited.
+an array of nodes, which will be merged into the parent array at the offset of the current node.
- * `NodeVisitor::REMOVE_NODE`, in which case the current node will be removed from the parent array.
+I.e. if in `array(A, B, C)` the node `B` should be replaced with `array(X, Y, Z)` the result will
- * `NodeVisitor::REPLACE_WITH_NULL`, in which case the current node will be replaced with `null`.
+be `array(A, X, Y, Z, C)`.
- * An array of nodes, which will be merged into the parent array at the offset of the current node.
-   I.e. if in `array(A, B, C)` the node `B` should be replaced with `array(X, Y, Z)` the result will
-   be `array(A, X, Y, Z, C)`.
 
 Instead of manually implementing the `NodeVisitor` interface you can also extend the `NodeVisitorAbstract`
 class, which will define empty default implementations for all the above methods.
@@ -386,16 +372,15 @@ unqualified function and constant names. These are resolved at runtime and thus
 know which function they are referring to. In most cases this is a non-issue as the global functions
 are meant.
 
-Additionally, the `NameResolver` adds a `namespacedName` subnode to class, function and constant
+Also the `NameResolver` adds a `namespacedName` subnode to class, function and constant declarations
-declarations that contains the namespaced name instead of only the shortname that is available via
+that contains the namespaced name instead of only the shortname that is available via `name`.
-`name`.
 
 > Read more: [Name resolution documentation](component/Name_resolution.markdown)
 
 Example: Converting namespaced code to pseudo namespaces
 --------------------------------------------------------
 
-A small example to understand the concept: We want to convert namespaced code to pseudo namespaces,
+A small example to understand the concept: We want to convert namespaced code to pseudo namespaces
 so it works on 5.2, i.e. names like `A\\B` should be converted to `A_B`. Note that such conversions
 are fairly complicated if you take PHP's dynamic features into account, so our conversion will
 assume that no dynamic features are used.
@@ -411,7 +396,7 @@ use PhpParser\NodeVisitor\NameResolver;
 $inDir  = '/some/path';
 $outDir = '/some/other/path';
 
-$parser        = (new ParserFactory())->createForNewestSupportedVersion();
+$parser        = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
 $traverser     = new NodeTraverser;
 $prettyPrinter = new PrettyPrinter\Standard;
 
@@ -447,7 +432,7 @@ foreach ($files as $file) {
 }
 ```
 
-Now lets start with the main code, the `NamespaceConverter`. One thing it needs to do
+Now lets start with the main code, the `NodeVisitor\NamespaceConverter`. One thing it needs to do
 is convert `A\\B` style names to `A_B` style ones.
 
 ```php
@@ -502,7 +487,7 @@ The last thing we need to do is remove the `namespace` and `use` statements:
 ```php
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
-use PhpParser\NodeVisitor;
+use PhpParser\NodeTraverser;
 
 class NodeVisitor_NamespaceConverter extends \PhpParser\NodeVisitorAbstract
 {
@@ -522,7 +507,7 @@ class NodeVisitor_NamespaceConverter extends \PhpParser\NodeVisitorAbstract
             return $node->stmts;
         } elseif ($node instanceof Stmt\Use_) {
             // remove use nodes altogether
-            return NodeVisitor::REMOVE_NODE;
+            return NodeTraverser::REMOVE_NODE;
         }
     }
 }

doc/README.md

@@ -27,8 +27,9 @@ Component documentation
   * [AST builders](component/AST_builders.markdown)
     * Fluent builders for AST nodes
   * [Lexer](component/Lexer.markdown)
-    * Emulation
+    * Lexer options
-    * Tokens, positions and attributes
+    * Token and file positions for nodes
+    * Custom attributes
   * [Error handling](component/Error_handling.markdown)
     * Column information for errors
     * Error recovery (parsing of syntactically incorrect code)

doc/component/AST_builders.markdown

@@ -15,10 +15,9 @@ accessed through `getNode()`. Fluent builders are available for
 the following syntactic elements:
 
  * namespaces and use statements
- * classes, interfaces, traits and enums
+ * classes, interfaces and traits
  * methods, functions and parameters
- * properties, class constants and enum cases
+ * properties
- * trait uses and trait use adaptations
 
 Here is an example:
 
@@ -96,13 +95,13 @@ abstract class SomeOtherClass extends SomeClass implements A\Few, \Interfaces
         AnotherTrait::func insteadof SecondTrait;
     }
     protected $someProperty;
-    private $anotherProperty = [1, 2, 3];
+    private $anotherProperty = array(1, 2, 3);
     /**
      * This method does something.
      *
      * @param SomeClass And takes a parameter
      */
-    abstract public function someMethod(SomeClass $someParam): bool;
+    public abstract function someMethod(SomeClass $someParam) : bool;
     protected function anotherMethod($someParam = 'test')
     {
         print $someParam;
@@ -134,8 +133,6 @@ nodes. The following methods are currently available:
  * `propertyFetch($var, $name)`: Creates a property fetch node. Converts `$name` to an `Identifier`
    node.
  * `concat(...$exprs)`: Create a tree of `BinaryOp\Concat` nodes for the given expressions.
- * `attribute($name, $args)`: Create a `Attribute` node. Converts `$name` to a `Name` node and
-   normalizes arguments.
 
 These methods may be expanded on an as-needed basis. Please open an issue or PR if a common
 operation is missing.

doc/component/Constant_expression_evaluation.markdown

@@ -19,9 +19,9 @@ PHP-Parser supports evaluation of such constant expressions through the `ConstEx
 
 use PhpParser\{ConstExprEvaluator, ConstExprEvaluationException};
 
-$evaluator = new ConstExprEvaluator();
+$evalutator = new ConstExprEvaluator();
 try {
-    $value = $evaluator->evaluateSilently($someExpr);
+    $value = $evalutator->evaluateSilently($someExpr);
 } catch (ConstExprEvaluationException $e) {
     // Either the expression contains unsupported expression types,
     // or an error occurred during evaluation
@@ -45,7 +45,7 @@ use PhpParser\Node\{Expr, Scalar};
 $evaluator = new ConstExprEvaluator();
 
 // 10 / 0
-$expr = new Expr\BinaryOp\Div(new Scalar\Int_(10), new Scalar\Int_(0));
+$expr = new Expr\BinaryOp\Div(new Scalar\LNumber(10), new Scalar\LNumber(0));
 
 var_dump($evaluator->evaluateDirectly($expr)); // float(INF)
 // Warning: Division by zero
@@ -69,8 +69,6 @@ expressions, apart from the following:
  * `Scalar\MagicConst\*`
  * `Expr\ConstFetch` (only null/false/true are handled)
  * `Expr\ClassConstFetch`
- * `Expr\New_` (since PHP 8.1)
- * `Expr\PropertyFetch` (since PHP 8.2)
 
 Handling these expression types requires non-local information, such as which global constants are
 defined. By default, the evaluator will throw a `ConstExprEvaluationException` when it encounters
@@ -85,7 +83,7 @@ specifying an evaluation fallback function:
 use PhpParser\{ConstExprEvaluator, ConstExprEvaluationException};
 use PhpParser\Node\Expr;
 
-$evaluator = new ConstExprEvaluator(function(Expr $expr) {
+$evalutator = new ConstExprEvaluator(function(Expr $expr) {
     if ($expr instanceof Expr\ConstFetch) {
         return fetchConstantSomehow($expr);
     }
@@ -98,7 +96,7 @@ $evaluator = new ConstExprEvaluator(function(Expr $expr) {
 });
 
 try {
-    $evaluator->evaluateSilently($someExpr);
+    $evalutator->evaluateSilently($someExpr);
 } catch (ConstExprEvaluationException $e) {
     // Handle exception
 }
@@ -114,4 +112,4 @@ class Test {
     const A = self::B;
     const B = self::A;
 }
-```
+```

doc/component/Error_handling.markdown

@@ -4,12 +4,29 @@ Error handling
 Errors during parsing or analysis are represented using the `PhpParser\Error` exception class. In addition to an error
 message, an error can also store additional information about the location the error occurred at.
 
-How much location information is available depends on the origin of the error. At a minimum the start line of the error
+How much location information is available depends on the origin of the error and how many lexer attributes have been
-is usually available.
+enabled. At a minimum the start line of the error is usually available.
 
 Column information
 ------------------
 
+In order to receive information about not only the line, but also the column span an error occurred at, the file
+position attributes in the lexer need to be enabled:
+
+```php
+$lexer = new PhpParser\Lexer(array(
+    'usedAttributes' => array('comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'),
+));
+$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::PREFER_PHP7, $lexer);
+
+try {
+    $stmts = $parser->parse($code);
+    // ...
+} catch (PhpParser\Error $e) {
+    // ...
+}
+```
+
 Before using column information, its availability needs to be checked with `$e->hasColumnInfo()`, as the precise
 location of an error cannot always be determined. The methods for retrieving column information also have to be passed
 the source code of the parsed file. An example for printing an error:
@@ -39,7 +56,7 @@ To instead collect all encountered errors into an array, while trying to continu
 an instance of `ErrorHandler\Collecting` can be passed to the `Parser::parse()` method. A usage example:
 
 ```php
-$parser = (new PhpParser\ParserFactory())->createForHostVersion();
+$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::ONLY_PHP7);
 $errorHandler = new PhpParser\ErrorHandler\Collecting;
 
 $stmts = $parser->parse($code, $errorHandler);
@@ -55,6 +72,4 @@ if (null !== $stmts) {
 }
 ```
 
-The partial AST may contain `Expr\Error` nodes that indicate that an error occurred while parsing an expression.
-
 The `NameResolver` visitor also accepts an `ErrorHandler` as a constructor argument.

doc/component/FAQ.markdown

@@ -16,9 +16,10 @@ use PhpParser\ParserFactory;
 
 $code = '...';
 
-$traverser = new NodeTraverser(new ParentConnectingVisitor);
+$traverser = new NodeTraverser;
+$traverser->addVisitor(new ParentConnectingVisitor);
 
-$parser = (new ParserFactory())->createForHostVersion();
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
 $ast    = $parser->parse($code);
 $ast    = $traverser->traverse($ast);
 ```
@@ -38,9 +39,10 @@ use PhpParser\ParserFactory;
 
 $code = '...';
 
-$traverser = new NodeTraverser(new NodeConnectingVisitor);
+$traverser = new NodeTraverser;
+$traverser->addVisitor(new NodeConnectingVisitor);
 
-$parser = (new ParserFactory())->createForHostVersion();
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
 $ast    = $parser->parse($code);
 $ast    = $traverser->traverse($ast);
 ```
@@ -51,17 +53,3 @@ obtained through `$node->getAttribute('next')`.
 
 `ParentConnectingVisitor` and `NodeConnectingVisitor` should not be used at the same time. The latter
 includes the functionality of the former.
-
-
-How can I limit the impact of cyclic references in the AST?
------
-
-NodeConnectingVisitor adds a parent reference, which introduces a cycle. This means that the AST can now only be collected by the cycle garbage collector.
-This in turn can lead to performance and/or memory issues. 
-
-To break the cyclic references between AST nodes `NodeConnectingVisitor` supports a boolean `$weakReferences` constructor parameter.
-When set to `true`, all attributes added by `NodeConnectingVisitor` will be wrapped into a `WeakReference` object.
-
-After enabling this parameter, the parent node can be obtained through `$node->getAttribute('weak_parent')`,
-the previous node can be obtained through `$node->getAttribute('weak_previous')`, and the next node can be
-obtained through `$node->getAttribute('weak_next')`.

doc/component/JSON_representation.markdown

@@ -18,7 +18,7 @@ function printLine($msg) {
 }
 CODE;
 
-$parser = (new ParserFactory())->createForHostVersion();
+$parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7);
 
 try {
     $stmts = $parser->parse($code);
@@ -35,6 +35,65 @@ This will result in the following output (which includes attributes):
 [
     {
         "nodeType": "Stmt_Function",
+        "byRef": false,
+        "name": {
+            "nodeType": "Identifier",
+            "name": "printLine",
+            "attributes": {
+                "startLine": 4,
+                "endLine": 4
+            }
+        },
+        "params": [
+            {
+                "nodeType": "Param",
+                "type": null,
+                "byRef": false,
+                "variadic": false,
+                "var": {
+                    "nodeType": "Expr_Variable",
+                    "name": "msg",
+                    "attributes": {
+                        "startLine": 4,
+                        "endLine": 4
+                    }
+                },
+                "default": null,
+                "attributes": {
+                    "startLine": 4,
+                    "endLine": 4
+                }
+            }
+        ],
+        "returnType": null,
+        "stmts": [
+            {
+                "nodeType": "Stmt_Echo",
+                "exprs": [
+                    {
+                        "nodeType": "Expr_Variable",
+                        "name": "msg",
+                        "attributes": {
+                            "startLine": 5,
+                            "endLine": 5
+                        }
+                    },
+                    {
+                        "nodeType": "Scalar_String",
+                        "value": "\n",
+                        "attributes": {
+                            "startLine": 5,
+                            "endLine": 5,
+                            "kind": 2
+                        }
+                    }
+                ],
+                "attributes": {
+                    "startLine": 5,
+                    "endLine": 5
+                }
+            }
+        ],
         "attributes": {
             "startLine": 4,
             "comments": [
@@ -42,79 +101,12 @@ This will result in the following output (which includes attributes):
                     "nodeType": "Comment_Doc",
                     "text": "\/** @param string $msg *\/",
                     "line": 3,
-                    "filePos": 7,
+                    "filePos": 9,
-                    "tokenPos": 2,
+                    "tokenPos": 2
-                    "endLine": 3,
-                    "endFilePos": 31,
-                    "endTokenPos": 2
                 }
             ],
             "endLine": 6
-        },
+        }
-        "byRef": false,
-        "name": {
-            "nodeType": "Identifier",
-            "attributes": {
-                "startLine": 4,
-                "endLine": 4
-            },
-            "name": "printLine"
-        },
-        "params": [
-            {
-                "nodeType": "Param",
-                "attributes": {
-                    "startLine": 4,
-                    "endLine": 4
-                },
-                "type": null,
-                "byRef": false,
-                "variadic": false,
-                "var": {
-                    "nodeType": "Expr_Variable",
-                    "attributes": {
-                        "startLine": 4,
-                        "endLine": 4
-                    },
-                    "name": "msg"
-                },
-                "default": null,
-                "flags": 0,
-                "attrGroups": []
-            }
-        ],
-        "returnType": null,
-        "stmts": [
-            {
-                "nodeType": "Stmt_Echo",
-                "attributes": {
-                    "startLine": 5,
-                    "endLine": 5
-                },
-                "exprs": [
-                    {
-                        "nodeType": "Expr_Variable",
-                        "attributes": {
-                            "startLine": 5,
-                            "endLine": 5
-                        },
-                        "name": "msg"
-                    },
-                    {
-                        "nodeType": "Scalar_String",
-                        "attributes": {
-                            "startLine": 5,
-                            "endLine": 5,
-                            "kind": 2,
-                            "rawValue": "\"\\n\""
-                        },
-                        "value": "\n"
-                    }
-                ]
-            }
-        ],
-        "attrGroups": [],
-        "namespacedName": null
     }
 ]
 ```

doc/component/Lexer.markdown

@@ -1,96 +1,57 @@
 Lexer component documentation
 =============================
 
-The lexer is responsible for providing tokens to the parser. Typical use of the library does not require direct
+The lexer is responsible for providing tokens to the parser. The project comes with two lexers: `PhpParser\Lexer` and
-interaction with the lexer, as an appropriate lexer is created by `PhpParser\ParserFactory`. The tokens produced
+`PhpParser\Lexer\Emulative`. The latter is an extension of the former, which adds the ability to emulate tokens of
-by the lexer can then be retrieved using `PhpParser\Parser::getTokens()`.
+newer PHP versions and thus allows parsing of new code on older versions.
 
-Emulation
+This documentation discusses options available for the default lexers and explains how lexers can be extended.
----------
 
-While this library implements a custom parser, it relies on PHP's `ext/tokenizer` extension to perform lexing. However,
+Lexer options
-this extension only supports lexing code for the PHP version you are running on, while this library also wants to support
+-------------
-parsing newer code. For that reason, the lexer performs additional "emulation" in three layers:
 
-First, PhpParser uses the `PhpToken` based representation introduced in PHP 8.0, rather than the array-based tokens from
+The two default lexers accept an `$options` array in the constructor. Currently only the `'usedAttributes'` option is
-previous versions. The `PhpParser\Token` class either extends `PhpToken` (on PHP 8.0) or a polyfill implementation. The
+supported, which allows you to specify which attributes will be added to the AST nodes. The attributes can then be
-polyfill implementation will also perform two emulations that are required by the parser and cannot be disabled:
+accessed using `$node->getAttribute()`, `$node->setAttribute()`, `$node->hasAttribute()` and `$node->getAttributes()`
-
+methods. A sample options array:
- * Single-line comments use the PHP 8.0 representation that does not include a trailing newline. The newline will be
-   part of a following `T_WHITESPACE` token.
- * Namespaced names use the PHP 8.0 representation using `T_NAME_FULLY_QUALIFIED`, `T_NAME_QUALIFIED` and
-   `T_NAME_RELATIVE` tokens, rather than the previous representation using a sequence of `T_STRING` and `T_NS_SEPARATOR`.
-   This means that certain code that is legal on older versions (namespaced names including whitespace, such as `A \ B`)
-   will not be accepted by the parser.
-
-Second, the `PhpParser\Lexer` base class will convert `&` tokens into the PHP 8.1 representation of either
-`T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG` or `T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG`. This is required by the parser
-and cannot be disabled.
-
-Finally, `PhpParser\Lexer\Emulative` performs other, optional emulations. This lexer is parameterized by `PhpVersion`
-and will try to emulate `ext/tokenizer` output for that version. This is done using separate `TokenEmulator`s for each
-emulated feature.
-
-Emulation is usually used to support newer PHP versions, but there is also very limited support for reverse emulation to
-older PHP versions, which can make keywords from newer versions non-reserved.
-
-Tokens, positions and attributes
---------------------------------
-
-The `Lexer::tokenize()` method returns an array of `PhpParser\Token`s. The most important parts of the interface can be
-summarized as follows:
 
 ```php
-class Token {
+$lexer = new PhpParser\Lexer(array(
-    /** @var int Token ID, either T_* or ord($char) for single-character tokens. */
+    'usedAttributes' => array(
-    public int $id;
+        'comments', 'startLine', 'endLine'
-    /** @var string The textual content of the token. */
+    )
-    public string $text;
+));
-    /** @var int The 1-based starting line of the token (or -1 if unknown). */
-    public int $line;
-    /** @var int The 0-based starting position of the token (or -1 if unknown). */
-    public int $pos;
-
-    /** @param int|string|(int|string)[] $kind Token ID or text (or array of them) */
-    public function is($kind): bool;
-}
 ```
 
-Unlike PHP's own `PhpToken::tokenize()` output, the token array is terminated by a sentinel token with ID 0.
+The attributes used in this example match the default behavior of the lexer. The following attributes are supported:
 
-The lexer is normally invoked implicitly by the parser. In that case, the tokens for the last parse can be retrieved
+ * `comments`: Array of `PhpParser\Comment` or `PhpParser\Comment\Doc` instances, representing all comments that occurred
-using `Parser::getTokens()`.
+   between the previous non-discarded token and the current one. Use of this attribute is required for the
-
+   `$node->getComments()` and `$node->getDocComment()` methods to work. The attribute is also needed if you wish the pretty
-Nodes in the AST produced by the parser always corresponds to some range of tokens. The parser adds a number of
+   printer to retain comments present in the original code.
-positioning attributes to allow mapping nodes back to lines, tokens or file offsets:
+ * `startLine`: Line in which the node starts. This attribute is required for the `$node->getLine()` to work. It is also
-
+   required if syntax errors should contain line number information.
- * `startLine`: Line in which the node starts. Used by `$node->getStartLine()`.
+ * `endLine`: Line in which the node ends. Required for `$node->getEndLine()`.
- * `endLine`: Line in which the node ends. Used by `$node->getEndLine()`.
+ * `startTokenPos`: Offset into the token array of the first token in the node. Required for `$node->getStartTokenPos()`.
- * `startTokenPos`: Offset into the token array of the first token in the node. Used by `$node->getStartTokenPos()`.
+ * `endTokenPos`: Offset into the token array of the last token in the node. Required for `$node->getEndTokenPos()`.
- * `endTokenPos`: Offset into the token array of the last token in the node. Used by `$node->getEndTokenPos()`.
+ * `startFilePos`: Offset into the code string of the first character that is part of the node. Required for `$node->getStartFilePos()`.
- * `startFilePos`: Offset into the code string of the first character that is part of the node. Used by `$node->getStartFilePos()`.
+ * `endFilePos`: Offset into the code string of the last character that is part of the node. Required for `$node->getEndFilePos()`.
- * `endFilePos`: Offset into the code string of the last character that is part of the node. Used by `$node->getEndFilePos()`.
-
-Note that `start`/`end` here are closed rather than half-open ranges. This means that a node consisting of a single
-token will have `startTokenPos == endTokenPos` rather than `startTokenPos + 1 == endTokenPos`. This also means that a
-zero-length node will have `startTokenPos -1 == endTokenPos`.
 
 ### Using token positions
 
 > **Note:** The example in this section is outdated in that this information is directly available in the AST: While
 > `$property->isPublic()` does not distinguish between `public` and `var`, directly checking `$property->flags` for
 > the `$property->flags & Class_::VISIBILITY_MODIFIER_MASK) === 0` allows making this distinction without resorting to
-> tokens. However, the general idea behind the example still applies in other cases.
+> tokens. However the general idea behind the example still applies in other cases.
 
 The token offset information is useful if you wish to examine the exact formatting used for a node. For example the AST
 does not distinguish whether a property was declared using `public` or using `var`, but you can retrieve this
 information based on the token position:
 
 ```php
-/** @param PhpParser\Token[] $tokens */
 function isDeclaredUsingVar(array $tokens, PhpParser\Node\Stmt\Property $prop) {
-    $i = $prop->getStartTokenPos();
+    $i = $prop->getAttribute('startTokenPos');
-    return $tokens[$i]->id === T_VAR;
+    return $tokens[$i][0] === T_VAR;
 }
 ```
 
@@ -111,16 +72,88 @@ class MyNodeVisitor extends PhpParser\NodeVisitorAbstract {
     }
 }
 
-$parser = (new PhpParser\ParserFactory())->createForHostVersion($lexerOptions);
+$lexer = new PhpParser\Lexer(array(
+    'usedAttributes' => array(
+        'comments', 'startLine', 'endLine', 'startTokenPos', 'endTokenPos'
+    )
+));
+$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::ONLY_PHP7, $lexer);
 
 $visitor = new MyNodeVisitor();
-$traverser = new PhpParser\NodeTraverser($visitor);
+$traverser = new PhpParser\NodeTraverser();
+$traverser->addVisitor($visitor);
 
 try {
     $stmts = $parser->parse($code);
-    $visitor->setTokens($parser->getTokens());
+    $visitor->setTokens($lexer->getTokens());
     $stmts = $traverser->traverse($stmts);
 } catch (PhpParser\Error $e) {
     echo 'Parse Error: ', $e->getMessage();
 }
 ```
+
+The same approach can also be used to perform specific modifications in the code, without changing the formatting in
+other places (which is the case when using the pretty printer).
+
+Lexer extension
+---------------
+
+A lexer has to define the following public interface:
+
+```php
+function startLexing(string $code, ErrorHandler $errorHandler = null): void;
+function getTokens(): array;
+function handleHaltCompiler(): string;
+function getNextToken(string &$value = null, array &$startAttributes = null, array &$endAttributes = null): int;
+```
+
+The `startLexing()` method is invoked whenever the `parse()` method of the parser is called and is passed the source
+code that is to be lexed (including the opening tag). It can be used to reset state or preprocess the source code or tokens. The
+passed `ErrorHandler` should be used to report lexing errors.
+
+The `getTokens()` method returns the current token array, in the usual `token_get_all()` format. This method is not
+used by the parser (which uses `getNextToken()`), but is useful in combination with the token position attributes.
+
+The `handleHaltCompiler()` method is called whenever a `T_HALT_COMPILER` token is encountered. It has to return the
+remaining string after the construct (not including `();`).
+
+The `getNextToken()` method returns the ID of the next token (as defined by the `Parser::T_*` constants). If no more
+tokens are available it must return `0`, which is the ID of the `EOF` token. Furthermore the string content of the
+token should be written into the by-reference `$value` parameter (which will then be available as `$n` in the parser).
+
+### Attribute handling
+
+The other two by-ref variables `$startAttributes` and `$endAttributes` define which attributes will eventually be
+assigned to the generated nodes: The parser will take the `$startAttributes` from the first token which is part of the
+node and the `$endAttributes` from the last token that is part of the node.
+
+E.g. if the tokens `T_FUNCTION T_STRING ... '{' ... '}'` constitute a node, then the `$startAttributes` from the
+`T_FUNCTION` token will be taken and the `$endAttributes` from the `'}'` token.
+
+An application of custom attributes is storing the exact original formatting of literals: While the parser does retain
+some information about the formatting of integers (like decimal vs. hexadecimal) or strings (like used quote type), it
+does not preserve the exact original formatting (e.g. leading zeros for integers or escape sequences in strings). This
+can be remedied by storing the original value in an attribute:
+
+```php
+use PhpParser\Lexer;
+use PhpParser\Parser\Tokens;
+
+class KeepOriginalValueLexer extends Lexer // or Lexer\Emulative
+{
+    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) {
+        $tokenId = parent::getNextToken($value, $startAttributes, $endAttributes);
+
+        if ($tokenId == Tokens::T_CONSTANT_ENCAPSED_STRING   // non-interpolated string
+            || $tokenId == Tokens::T_ENCAPSED_AND_WHITESPACE // interpolated string
+            || $tokenId == Tokens::T_LNUMBER                 // integer
+            || $tokenId == Tokens::T_DNUMBER                 // floating point number
+        ) {
+            // could also use $startAttributes, doesn't really matter here
+            $endAttributes['originalValue'] = $value;
+        }
+
+        return $tokenId;
+    }
+}
+```

doc/component/Name_resolution.markdown

@@ -10,7 +10,7 @@ visitor (NameResolver) based on it.
 The NameResolver visitor
 ------------------------
 
-The `NameResolver` visitor can (and for nearly all uses of the AST, should) be applied to resolve names
+The `NameResolver` visitor can (and for nearly all uses of the AST, is) be applied to resolve names
 to their fully-qualified form, to the degree that this is possible.
 
 ```php
@@ -24,7 +24,7 @@ $stmts = $nodeTraverser->traverse($stmts);
 
 In the default configuration, the name resolver will perform three actions:
 
- * Declarations of functions, classes, interfaces, traits, enums and global constants will have a
+ * Declarations of functions, classes, interfaces, traits and global constants will have a
    `namespacedName` property added, which contains the function/class/etc name including the
    namespace prefix. For historic reasons this is a **property** rather than an attribute.
  * Names will be replaced by fully qualified resolved names, which are instances of
@@ -32,7 +32,7 @@ In the default configuration, the name resolver will perform three actions:
  * Unqualified function and constant names inside a namespace cannot be statically resolved. Inside
    a namespace `Foo`, a call to `strlen()` may either refer to the namespaced `\Foo\strlen()`, or
    the global `\strlen()`. Because PHP-Parser does not have the necessary context to decide this,
-   such names are left unresolved. Additionally, a `namespacedName` **attribute** is added to the
+   such names are left unresolved. Additionally a `namespacedName` **attribute** is added to the
    name node.
 
 The name resolver accepts an option array as the second argument, with the following default values:
@@ -47,13 +47,13 @@ $nameResolver = new PhpParser\NodeVisitor\NameResolver(null, [
 If the `preserveOriginalNames` option is enabled, then the resolved (fully qualified) name will have
 an `originalName` attribute, which contains the unresolved name.
 
-If the `replaceNodes` option is disabled, then names will no longer be resolved in-place. Instead, a
+If the `replaceNodes` option is disabled, then names will no longer be resolved in-place. Instead a
 `resolvedName` attribute will be added to each name, which contains the resolved (fully qualified)
 name. Once again, if an unqualified function or constant name cannot be resolved, then the
 `resolvedName` attribute will not be present, and instead a `namespacedName` attribute is added.
 
 The `replaceNodes` attribute is useful if you wish to perform modifications on the AST, as you
-probably do not wish the resulting code to have fully resolved names as a side-effect.
+probably do not wish the resoluting code to have fully resolved names as a side-effect.
 
 The NameContext
 ---------------
@@ -75,7 +75,7 @@ class NameContext {
 }
 ```
 
-The `$type` parameters accept one of the `Stmt\Use_::TYPE_*` constants, which represent the three
+The `$type` parameters accept on of the `Stmt\Use_::TYPE_*` constants, which represent the three
 basic symbol types in PHP (functions, constants and everything else).
 
 Next to name resolution, the `NameContext` also supports the reverse operation of finding a short
@@ -84,4 +84,4 @@ representation of a name given the current name resolution environment.
 The name context is intended to be used for name resolution operations outside the AST itself, such
 as class names inside doc comments. A visitor running in parallel with the name resolver can access
 the name context using `$nameResolver->getNameContext()`. Alternatively a visitor can use an
-independent context and explicitly feed `Namespace` and `Use` nodes to it.
+independent context and explicitly feed `Namespace` and `Use` nodes to it.

doc/component/Performance.markdown

@@ -40,5 +40,26 @@ parse multiple files.
 
 When possible, objects should be reused rather than being newly instantiated for every use. Some
 objects have expensive initialization procedures, which will be unnecessarily repeated if the object
-is not reused. (Currently two objects with particularly expensive setup are parsers and pretty
+is not reused. (Currently two objects with particularly expensive setup are lexers and pretty
 printers, though the details might change between versions of this library.)
+
+Garbage collection
+------------------
+
+A limitation in PHP's cyclic garbage collector may lead to major performance degradation when the
+active working set exceeds 10000 objects (or arrays). Especially when parsing very large files this
+limit is significantly exceeded and PHP will spend the majority of time performing unnecessary
+garbage collection attempts.
+
+Without GC, parsing time is roughly linear in the input size. With GC, this degenerates to quadratic
+runtime for large files. While the specifics may differ, as a rough guideline you may expect a 2.5x
+GC overhead for 500KB files and a 5x overhead for 1MB files.
+
+Because this a limitation in PHP's implementation, there is no easy way to work around this. If
+possible, you should avoid parsing very large files, as they will impact overall execution time
+disproportionally (and are usually generated anyway).
+
+Of course, you can also try to (temporarily) disable GC. By design the AST generated by PHP-Parser
+is cycle-free, so the AST itself will never cause leaks with GC disabled. However, other code
+(including for example the parser object itself) may hold cycles, so disabling of GC should be
+approached with care.

doc/component/Pretty_printing.markdown

@@ -31,30 +31,10 @@ expression.
 Customizing the formatting
 --------------------------
 
-The pretty printer respects a number of `kind` attributes used by some nodes (e.g., whether an
+Apart from an `shortArraySyntax` option, the default pretty printer does not provide any
-integer should be printed as decimal, hexadecimal, etc). Additionally, it supports three options:
+functionality to customize the formatting of the generated code. The pretty printer does respect a
-
+number of `kind` attributes used by some notes (e.g., whether an integer should be printed as
-* `phpVersion` (defaults to 7.4) allows opting into formatting that is not supported by older PHP
+decimal, hexadecimal, etc), but there are no options to control brace placement or similar.
-  versions.
-* `newline` (defaults to `"\n"`) can be set to `"\r\n"` in order to produce Windows newlines.
-* `indent` (defaults to four spaces `"    "`) can be set to any number of spaces or a single tab.
-* `shortArraySyntax` determines the used array syntax if the `kind` attribute is not set. This is
-  a legacy option, and `phpVersion` should be used to control this behavior instead.
-
-The behaviors controlled by `phpVersion` (defaults to PHP 7.4) are:
-
-* For PHP >= 7.0, short array syntax `[]` will be used by default. This does not affect nodes that
-  specify an explicit array syntax using the `kind` attribute.
-* For PHP >= 7.0, parentheses around `yield` expressions will only be printed when necessary.
-* For PHP >= 7.1, the short array syntax `[]` will be used for destructuring by default (instead of
-  `list()`). This does not affect nodes that specify and explicit syntax using the `kind` attribute.
-* For PHP >= 7.3, a newline is no longer forced after heredoc/nowdoc strings, as the requirement
-  for this has been removed with the introduction of flexible heredoc/nowdoc strings.
-* For PHP >= 7.3, heredoc/nowdoc strings are indented just like regular code. This was allowed with
-  the introduction of flexible heredoc/nowdoc strings.
-
-The default pretty printer does not provide functionality for fine-grained customization of code
-formatting.
 
 If you want to make minor changes to the formatting, the easiest way is to extend the pretty printer
 and override the methods responsible for the node types you are interested in.
@@ -66,37 +46,51 @@ default pretty printer with an existing library for code reformatting, such as
 Formatting-preserving pretty printing
 -------------------------------------
 
+> **Note:** This functionality is **experimental** and not yet complete.
+
 For automated code refactoring, migration and similar, you will usually only want to modify a small
 portion of the code and leave the remainder alone. The basic pretty printer is not suitable for
 this, because it will also reformat parts of the code which have not been modified.
 
-Since PHP-Parser 4.0, a formatting-preserving pretty-printing mode is available, which
+Since PHP-Parser 4.0, an experimental formatting-preserving pretty-printing mode is available, which
 attempts to preserve the formatting of code (those AST nodes that have not changed) and only reformat
 code which has been modified or newly inserted.
 
 Use of the formatting-preservation functionality requires some additional preparatory steps:
 
 ```php
-use PhpParser\{NodeTraverser, NodeVisitor, ParserFactory, PrettyPrinter};
+use PhpParser\{Lexer, NodeTraverser, NodeVisitor, Parser, PrettyPrinter};
+
+$lexer = new Lexer\Emulative([
+    'usedAttributes' => [
+        'comments',
+        'startLine', 'endLine',
+        'startTokenPos', 'endTokenPos',
+    ],
+]);
+$parser = new Parser\Php7($lexer);
+
+$traverser = new NodeTraverser();
+$traverser->addVisitor(new NodeVisitor\CloningVisitor());
+
+$printer = new PrettyPrinter\Standard();
 
-$parser = (new ParserFactory())->createForHostVersion();
 $oldStmts = $parser->parse($code);
-$oldTokens = $parser->getTokens();
+$oldTokens = $lexer->getTokens();
 
-// Run CloningVisitor before making changes to the AST.
-$traverser = new NodeTraverser(new NodeVisitor\CloningVisitor());
 $newStmts = $traverser->traverse($oldStmts);
 
 // MODIFY $newStmts HERE
 
-$printer = new PrettyPrinter\Standard();
 $newCode = $printer->printFormatPreserving($newStmts, $oldStmts, $oldTokens);
 ```
 
 If you make use of the name resolution functionality, you will likely want to disable the
-`replaceNodes` option. This will add resolved names as attributes, instead of directly modifying
+`replaceNodes` option. This will add resolved names as attributes, instead of directlying modifying
 the AST and causing spurious changes to the pretty printed code. For more information, see the
 [name resolution documentation](Name_resolution.markdown).
 
-The formatting-preservation works on a best-effort basis and may sometimes reformat more code than
+This functionality is experimental and not yet fully implemented. It should not provide incorrect
-necessary. If you encounter problems while using this functionality, please open an issue.
+code, but it may sometimes reformat more code than necessary. Open issues are tracked in
+[issue #344](https://github.com/nikic/PHP-Parser/issues/344). If you encounter problems while using
+this functionality, please open an issue, so we know what to prioritize.

doc/component/Walking_the_AST.markdown

@@ -11,7 +11,7 @@ use PhpParser\{Node, NodeTraverser, NodeVisitorAbstract};
 $traverser = new NodeTraverser;
 $traverser->addVisitor(new class extends NodeVisitorAbstract {
     public function leaveNode(Node $node) {
-        if ($node instanceof Node\Scalar\Int_) {
+        if ($node instanceof Node\Scalar\LNumber) {
             return new Node\Scalar\String_((string) $node->value);
         }
     }
@@ -21,18 +21,6 @@ $stmts = ...;
 $modifiedStmts = $traverser->traverse($stmts);
 ```
 
-Visitors can be either passed to the `NodeTraverser` constructor, or added using `addVisitor()`:
-
-```php
-$traverser = new NodeTraverser($visitor1, $visitor2, $visitor3);
-
-// Equivalent to:
-$traverser = new NodeTraverser();
-$traverser->addVisitor($visitor1);
-$traverser->addVisitor($visitor2);
-$traverser->addVisitor($visitor3);
-```
-
 Node visitors
 -------------
 
@@ -59,19 +47,20 @@ For example, if we have the following excerpt of an AST
 
 ```
 Expr_FuncCall(
-   name: Name(
+    name: Name(
-       name: printLine
+        parts: array(
-   )
+            0: printLine
-   args: array(
+        )
-       0: Arg(
+    )
-           name: null
+    args: array(
-           value: Scalar_String(
+        0: Arg(
-               value: Hello World!!!
+            value: Scalar_String(
-           )
+                value: Hello World!!!
-           byRef: false
+            )
-           unpack: false
+            byRef: false
-       )
+            unpack: false
-   )
+        )
+    )
 )
 ```
 
@@ -140,13 +129,14 @@ Now `$a && $b` will be replaced by `!($a && $b)`. Then the traverser will go int
 only) child of `!($a && $b)`, which is `$a && $b`. The transformation applies again and we end up
 with `!!($a && $b)`. This will continue until PHP hits the memory limit.
 
-Finally, there are three special replacement types. The first is removal of a node:
+Finally, two special replacement types are supported only by leaveNode. The first is removal of a
+node:
 
 ```php
 public function leaveNode(Node $node) {
     if ($node instanceof Node\Stmt\Return_) {
         // Remove all return statements
-        return NodeVisitor::REMOVE_NODE;
+        return NodeTraverser::REMOVE_NODE;
     }
 }
 ```
@@ -166,7 +156,7 @@ public function leaveNode(Node $node) {
         && $node->expr->name instanceof Node\Name
         && $node->expr->name->toString() === 'var_dump'
     ) {
-        return NodeVisitor::REMOVE_NODE;
+        return NodeTraverser::REMOVE_NODE;
     }
 }
 ```
@@ -175,22 +165,8 @@ This example will remove all calls to `var_dump()` which occur as expression sta
 that `var_dump($a);` will be removed, but `if (var_dump($a))` will not be removed (and there is no
 obvious way in which it can be removed).
 
-Another way to remove nodes is to replace them with `null`. For example, all `else` statements could
+Next to removing nodes, it is also possible to replace one node with multiple nodes. Again, this
-be removed as follows:
+only works inside leaveNode and only if the parent structure is an array.
-
-```php
-public function leaveNode(Node $node) {
-    if ($node instanceof Node\Stmt\Else_) {
-        return NodeVisitor::REPLACE_WITH_NULL;
-    }
-}
-```
-
-This is only safe to do if the subnode the node is stored in is nullable. `Node\Stmt\Else_` only
-occurs inside `Node\Stmt\If_::$else`, which is nullable, so this particular replacement is safe.
-
-Next to removing nodes, it is also possible to replace one node with multiple nodes. This
-only works if the parent structure is an array.
 
 ```php
 public function leaveNode(Node $node) {
@@ -222,7 +198,7 @@ private $classes = [];
 public function enterNode(Node $node) {
     if ($node instanceof Node\Stmt\Class_) {
         $this->classes[] = $node;
-        return NodeVisitor::DONT_TRAVERSE_CHILDREN;
+        return NodeTraverser::DONT_TRAVERSE_CHILDREN;
     }
 }
 ```
@@ -242,7 +218,7 @@ public function enterNode(Node $node) {
         $node->namespacedName->toString() === 'Foo\Bar\Baz'
     ) {
         $this->class = $node;
-        return NodeVisitor::STOP_TRAVERSAL;
+        return NodeTraverser::STOP_TRAVERSAL;
     }
 }
 ```
@@ -280,14 +256,13 @@ $visitorA->enterNode(Stmt_Return)
 $visitorB->enterNode(Stmt_Return)
 $visitorA->enterNode(Expr_Variable)
 $visitorB->enterNode(Expr_Variable)
-$visitorB->leaveNode(Expr_Variable)
 $visitorA->leaveNode(Expr_Variable)
-$visitorB->leaveNode(Stmt_Return)
+$visitorB->leaveNode(Expr_Variable)
 $visitorA->leaveNode(Stmt_Return)
+$visitorB->leaveNode(Stmt_Return)
 ```
 
-That is, when visiting a node, `enterNode()` and `leaveNode()` will always be called for all
+That is, when visiting a node, enterNode and leaveNode will always be called for all visitors.
-visitors, with the `leaveNode()` calls happening in the reverse order of the `enterNode()` calls.
 Running multiple visitors in parallel improves performance, as the AST only has to be traversed
 once. However, it is not always possible to write visitors in a way that allows interleaved
 execution. In this case, you can always fall back to performing multiple traversals:
@@ -312,7 +287,6 @@ special enterNode/leaveNode return values:
  * If a visitor returns a replacement node, subsequent visitors will be passed the replacement node,
    not the original one.
  * If a visitor returns `REMOVE_NODE`, subsequent visitors will not see this node.
- * If a visitor returns `REPLACE_WITH_NULL`, subsequent visitors will not see this node.
  * If a visitor returns an array of replacement nodes, subsequent visitors will see neither the node
    that was replaced, nor the replacement nodes.
 
@@ -359,6 +333,5 @@ be accessed: From parents to children. However, it can often be convenient to op
 reverse direction: When working on a node, you might want to check if the parent node satisfies a
 certain property.
 
-PHP-Parser does not add parent (or sibling) references to nodes by default, but you can enable them
+PHP-Parser does not add parent (or sibling) references to nodes by itself, but you can easily
-using the `ParentConnectingVisitor` or `NodeConnectingVisitor`. See the [FAQ](FAQ.markdown) for
+emulate this with a visitor. See the [FAQ](FAQ.markdown) for more information.
-more information.

grammar/README.md

@@ -1,8 +1,11 @@
 What do all those files mean?
 =============================
 
- * `php.y`:              PHP 5-8 grammar written in a pseudo language
+ * `php5.y`:             PHP 5 grammar written in a pseudo language
+ * `php7.y`:             PHP 7 grammar written in a pseudo language
+ * `tokens.y`:           Tokens definition shared between PHP 5 and PHP 7 grammars
  * `parser.template`:    A `kmyacc` parser prototype file for PHP
+ * `tokens.template`:    A `kmyacc` prototype file for the `Tokens` class
  * `rebuildParsers.php`: Preprocesses the grammar and builds the parser using `kmyacc`
 
 .phpy pseudo language

grammar/parser.template

@@ -1,14 +1,13 @@
-<?php declare(strict_types=1);
+<?php
 $meta #
-#semval($) $self->semValue
+#semval($) $this->semValue
-#semval($,%t) $self->semValue
+#semval($,%t) $this->semValue
 #semval(%n) $stackPos-(%l-%n)
 #semval(%n,%t) $stackPos-(%l-%n)
 
 namespace PhpParser\Parser;
 
 use PhpParser\Error;
-use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Name;
@@ -18,90 +17,88 @@ use PhpParser\Node\Stmt;
 
 /* This is an automatically GENERATED file, which should not be manually edited.
  * Instead edit one of the following:
- *  * the grammar file grammar/php.y
+ *  * the grammar files grammar/php5.y or grammar/php7.y
  *  * the skeleton file grammar/parser.template
  *  * the preprocessing script grammar/rebuildParsers.php
  */
 class #(-p) extends \PhpParser\ParserAbstract
 {
-#tokenval
+    protected $tokenToSymbolMapSize = #(YYMAXLEX);
-    public const %s = %n;
+    protected $actionTableSize = #(YYLAST);
-#endtokenval
+    protected $gotoTableSize = #(YYGLAST);
 
-    protected int $tokenToSymbolMapSize = #(YYMAXLEX);
+    protected $invalidSymbol = #(YYBADCH);
-    protected int $actionTableSize = #(YYLAST);
+    protected $errorSymbol = #(YYINTERRTOK);
-    protected int $gotoTableSize = #(YYGLAST);
+    protected $defaultAction = #(YYDEFAULT);
+    protected $unexpectedTokenRule = #(YYUNEXPECTED);
 
-    protected int $invalidSymbol = #(YYBADCH);
+    protected $YY2TBLSTATE = #(YY2TBLSTATE);
-    protected int $errorSymbol = #(YYINTERRTOK);
+    protected $numNonLeafStates = #(YYNLSTATES);
-    protected int $defaultAction = #(YYDEFAULT);
-    protected int $unexpectedTokenRule = #(YYUNEXPECTED);
 
-    protected int $YY2TBLSTATE = #(YY2TBLSTATE);
+    protected $symbolToName = array(
-    protected int $numNonLeafStates = #(YYNLSTATES);
-
-    protected array $symbolToName = array(
         #listvar terminals
     );
 
-    protected array $tokenToSymbol = array(
+    protected $tokenToSymbol = array(
         #listvar yytranslate
     );
 
-    protected array $action = array(
+    protected $action = array(
         #listvar yyaction
     );
 
-    protected array $actionCheck = array(
+    protected $actionCheck = array(
         #listvar yycheck
     );
 
-    protected array $actionBase = array(
+    protected $actionBase = array(
         #listvar yybase
     );
 
-    protected array $actionDefault = array(
+    protected $actionDefault = array(
         #listvar yydefault
     );
 
-    protected array $goto = array(
+    protected $goto = array(
         #listvar yygoto
     );
 
-    protected array $gotoCheck = array(
+    protected $gotoCheck = array(
         #listvar yygcheck
     );
 
-    protected array $gotoBase = array(
+    protected $gotoBase = array(
         #listvar yygbase
     );
 
-    protected array $gotoDefault = array(
+    protected $gotoDefault = array(
         #listvar yygdefault
     );
 
-    protected array $ruleToNonTerminal = array(
+    protected $ruleToNonTerminal = array(
         #listvar yylhs
     );
 
-    protected array $ruleToLength = array(
+    protected $ruleToLength = array(
         #listvar yylen
     );
 #if -t
 
-    protected array $productions = array(
+    protected $productions = array(
         #production-strings;
     );
 #endif
 
-    protected function initReduceCallbacks(): void {
+    protected function initReduceCallbacks() {
         $this->reduceCallbacks = [
 #reduce
-            %n => static function ($self, $stackPos) {
+            %n => function ($stackPos) {
                 %b
             },
 #noact
-            %n => null,
+            %n => function ($stackPos) {
+                $this->semValue = $this->semStack[$stackPos];
+            },
 #endreduce
         ];
     }

grammar/phpyLang.php

@@ -1,4 +1,4 @@
-<?php declare(strict_types=1);
+<?php
 
 ///////////////////////////////
 /// Utility regex constants ///
@@ -23,7 +23,6 @@ function preprocessGrammar($code) {
     $code = resolveNodes($code);
     $code = resolveMacros($code);
     $code = resolveStackAccess($code);
-    $code = str_replace('$this', '$self', $code);
 
     return $code;
 }
@@ -31,7 +30,7 @@ function preprocessGrammar($code) {
 function resolveNodes($code) {
     return preg_replace_callback(
         '~\b(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
-        function ($matches) {
+        function($matches) {
             // recurse
             $matches['params'] = resolveNodes($matches['params']);
 
@@ -54,7 +53,7 @@ function resolveNodes($code) {
 function resolveMacros($code) {
     return preg_replace_callback(
         '~\b(?<!::|->)(?!array\()(?<name>[a-z][A-Za-z]++)' . ARGS . '~',
-        function ($matches) {
+        function($matches) {
             // recurse
             $matches['args'] = resolveMacros($matches['args']);
 
@@ -66,13 +65,13 @@ function resolveMacros($code) {
 
             if ('attributes' === $name) {
                 assertArgs(0, $args, $name);
-                return '$this->getAttributes($this->tokenStartStack[#1], $this->tokenEndStack[$stackPos])';
+                return '$this->startAttributeStack[#1] + $this->endAttributes';
             }
 
             if ('stackAttributes' === $name) {
                 assertArgs(1, $args, $name);
-                return '$this->getAttributes($this->tokenStartStack[' . $args[0] . '], '
+                return '$this->startAttributeStack[' . $args[0] . ']'
-                       . ' $this->tokenEndStack[' . $args[0] . '])';
+                       . ' + $this->endAttributeStack[' . $args[0] . ']';
             }
 
             if ('init' === $name) {
@@ -88,15 +87,14 @@ function resolveMacros($code) {
             if ('pushNormalizing' === $name) {
                 assertArgs(2, $args, $name);
 
-                return 'if (' . $args[1] . ' !== null) { ' . $args[0] . '[] = ' . $args[1] . '; } $$ = ' . $args[0] . ';';
+                return 'if (is_array(' . $args[1] . ')) { $$ = array_merge(' . $args[0] . ', ' . $args[1] . '); }'
+                       . ' else { ' . $args[0] . '[] = ' . $args[1] . '; $$ = ' . $args[0] . '; }';
             }
 
-            if ('toBlock' == $name) {
+            if ('toArray' == $name) {
                 assertArgs(1, $args, $name);
 
-                return 'if (' . $args[0] . ' instanceof Stmt\Block) { $$ = ' . $args[0] . '->stmts; } '
+                return 'is_array(' . $args[0] . ') ? ' . $args[0] . ' : array(' . $args[0] . ')';
-                     . 'else if (' . $args[0] . ' === null) { $$ = []; } '
-                     . 'else { $$ = [' . $args[0] . ']; }';
             }
 
             if ('parseVar' === $name) {
@@ -108,20 +106,35 @@ function resolveMacros($code) {
             if ('parseEncapsed' === $name) {
                 assertArgs(3, $args, $name);
 
-                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\InterpolatedStringPart) {'
+                return 'foreach (' . $args[0] . ' as $s) { if ($s instanceof Node\Scalar\EncapsedStringPart) {'
                        . ' $s->value = Node\Scalar\String_::parseEscapeSequences($s->value, ' . $args[1] . ', ' . $args[2] . '); } }';
             }
 
             if ('makeNop' === $name) {
-                assertArgs(1, $args, $name);
+                assertArgs(3, $args, $name);
 
-                return $args[0] . ' = $this->maybeCreateNop($this->tokenStartStack[#1], $this->tokenEndStack[$stackPos])';
+                return '$startAttributes = ' . $args[1] . ';'
+                       . ' if (isset($startAttributes[\'comments\']))'
+                       . ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
+                       . ' else { ' . $args[0] . ' = null; }';
             }
 
             if ('makeZeroLengthNop' == $name) {
+                assertArgs(2, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                       . ' if (isset($startAttributes[\'comments\']))'
+                       . ' { ' . $args[0] . ' = new Stmt\Nop($this->createCommentNopAttributes($startAttributes[\'comments\'])); }'
+                       . ' else { ' . $args[0] . ' = null; }';
+            }
+
+            if ('prependLeadingComments' === $name) {
                 assertArgs(1, $args, $name);
 
-                return $args[0] . ' = $this->maybeCreateZeroLengthNop($this->tokenPos);';
+                return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
+                       . 'if (!empty($attrs[\'comments\'])) {'
+                       . '$stmts[0]->setAttribute(\'comments\', '
+                       . 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
             }
 
             return $matches[0];

grammar/rebuildParsers.php

@@ -1,17 +1,19 @@
-<?php declare(strict_types=1);
+<?php
 
 require __DIR__ . '/phpyLang.php';
 
-$parserToDefines = [
+$grammarFileToName = [
-    'Php7' => ['PHP7' => true],
+    __DIR__ . '/php5.y' => 'Php5',
-    'Php8' => ['PHP8' => true],
+    __DIR__ . '/php7.y' => 'Php7',
 ];
 
-$grammarFile    = __DIR__ . '/php.y';
+$tokensFile     = __DIR__ . '/tokens.y';
+$tokensTemplate = __DIR__ . '/tokens.template';
 $skeletonFile   = __DIR__ . '/parser.template';
 $tmpGrammarFile = __DIR__ . '/tmp_parser.phpy';
 $tmpResultFile  = __DIR__ . '/tmp_parser.php';
 $resultDir = __DIR__ . '/../lib/PhpParser/Parser';
+$tokensResultsFile = $resultDir . '/Tokens.php';
 
 $kmyacc = getenv('KMYACC');
 if (!$kmyacc) {
@@ -27,11 +29,13 @@ $optionKeepTmpGrammar = isset($options['--keep-tmp-grammar']);
 /// Main script ///
 ///////////////////
 
-foreach ($parserToDefines as $name => $defines) {
+$tokens = file_get_contents($tokensFile);
+
+foreach ($grammarFileToName as $grammarFile => $name) {
     echo "Building temporary $name grammar file.\n";
 
     $grammarCode = file_get_contents($grammarFile);
-    $grammarCode = replaceIfBlocks($grammarCode, $defines);
+    $grammarCode = str_replace('%tokens', $tokens, $grammarCode);
     $grammarCode = preprocessGrammar($grammarCode);
 
     file_put_contents($tmpGrammarFile, $grammarCode);
@@ -48,6 +52,10 @@ foreach ($parserToDefines as $name => $defines) {
     file_put_contents("$resultDir/$name.php", $resultCode);
     unlink($tmpResultFile);
 
+    echo "Building token definition.\n";
+    $output = execCmd("$kmyacc -m $tokensTemplate $tmpGrammarFile");
+    rename($tmpResultFile, $tokensResultsFile);
+
     if (!$optionKeepTmpGrammar) {
         unlink($tmpGrammarFile);
     }
@@ -64,17 +72,10 @@ function ensureDirExists($dir) {
 }
 
 function execCmd($cmd) {
-    $output = trim(shell_exec("$cmd 2>&1") ?? '');
+    $output = trim(shell_exec("$cmd 2>&1"));
     if ($output !== "") {
         echo "> " . $cmd . "\n";
         echo $output;
     }
     return $output;
 }
-
-function replaceIfBlocks(string $code, array $defines): string {
-    return preg_replace_callback('/\n#if\s+(\w+)\n(.*?)\n#endif/s', function ($matches) use ($defines) {
-        $value = $defines[$matches[1]] ?? false;
-        return $value ? $matches[2] : '';
-    }, $code);
-}

grammar/tokens.template

@@ -0,0 +1,17 @@
+<?php
+$meta #
+#semval($) $this->semValue
+#semval($,%t) $this->semValue
+#semval(%n) $this->stackPos-(%l-%n)
+#semval(%n,%t) $this->stackPos-(%l-%n)
+
+namespace PhpParser\Parser;
+#include;
+
+/* GENERATED file based on grammar/tokens.y */
+final class Tokens
+{
+#tokenval
+    const %s = %n;
+#endtokenval
+}

grammar/tokens.y

@@ -0,0 +1,115 @@
+/* We currently rely on the token ID mapping to be the same between PHP 5 and PHP 7 - so the same lexer can be used for
+ * both. This is enforced by sharing this token file. */
+
+%right T_THROW
+%left T_INCLUDE T_INCLUDE_ONCE T_EVAL T_REQUIRE T_REQUIRE_ONCE
+%left ','
+%left T_LOGICAL_OR
+%left T_LOGICAL_XOR
+%left T_LOGICAL_AND
+%right T_PRINT
+%right T_YIELD
+%right T_DOUBLE_ARROW
+%right T_YIELD_FROM
+%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL T_COALESCE_EQUAL
+%left '?' ':'
+%right T_COALESCE
+%left T_BOOLEAN_OR
+%left T_BOOLEAN_AND
+%left '|'
+%left '^'
+%left T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
+%nonassoc T_IS_EQUAL T_IS_NOT_EQUAL T_IS_IDENTICAL T_IS_NOT_IDENTICAL T_SPACESHIP
+%nonassoc '<' T_IS_SMALLER_OR_EQUAL '>' T_IS_GREATER_OR_EQUAL
+%left T_SL T_SR
+%left '+' '-' '.'
+%left '*' '/' '%'
+%right '!'
+%nonassoc T_INSTANCEOF
+%right '~' T_INC T_DEC T_INT_CAST T_DOUBLE_CAST T_STRING_CAST T_ARRAY_CAST T_OBJECT_CAST T_BOOL_CAST T_UNSET_CAST '@'
+%right T_POW
+%right '['
+%nonassoc T_NEW T_CLONE
+%token T_EXIT
+%token T_IF
+%left T_ELSEIF
+%left T_ELSE
+%left T_ENDIF
+%token T_LNUMBER
+%token T_DNUMBER
+%token T_STRING
+%token T_STRING_VARNAME
+%token T_VARIABLE
+%token T_NUM_STRING
+%token T_INLINE_HTML
+%token T_ENCAPSED_AND_WHITESPACE
+%token T_CONSTANT_ENCAPSED_STRING
+%token T_ECHO
+%token T_DO
+%token T_WHILE
+%token T_ENDWHILE
+%token T_FOR
+%token T_ENDFOR
+%token T_FOREACH
+%token T_ENDFOREACH
+%token T_DECLARE
+%token T_ENDDECLARE
+%token T_AS
+%token T_SWITCH
+%token T_MATCH
+%token T_ENDSWITCH
+%token T_CASE
+%token T_DEFAULT
+%token T_BREAK
+%token T_CONTINUE
+%token T_GOTO
+%token T_FUNCTION
+%token T_FN
+%token T_CONST
+%token T_RETURN
+%token T_TRY
+%token T_CATCH
+%token T_FINALLY
+%token T_THROW
+%token T_USE
+%token T_INSTEADOF
+%token T_GLOBAL
+%right T_STATIC T_ABSTRACT T_FINAL T_PRIVATE T_PROTECTED T_PUBLIC T_READONLY
+%token T_VAR
+%token T_UNSET
+%token T_ISSET
+%token T_EMPTY
+%token T_HALT_COMPILER
+%token T_CLASS
+%token T_TRAIT
+%token T_INTERFACE
+%token T_ENUM
+%token T_EXTENDS
+%token T_IMPLEMENTS
+%token T_OBJECT_OPERATOR
+%token T_NULLSAFE_OBJECT_OPERATOR
+%token T_DOUBLE_ARROW
+%token T_LIST
+%token T_ARRAY
+%token T_CALLABLE
+%token T_CLASS_C
+%token T_TRAIT_C
+%token T_METHOD_C
+%token T_FUNC_C
+%token T_LINE
+%token T_FILE
+%token T_START_HEREDOC
+%token T_END_HEREDOC
+%token T_DOLLAR_OPEN_CURLY_BRACES
+%token T_CURLY_OPEN
+%token T_PAAMAYIM_NEKUDOTAYIM
+%token T_NAMESPACE
+%token T_NS_C
+%token T_DIR
+%token T_NS_SEPARATOR
+%token T_ELLIPSIS
+%token T_NAME_FULLY_QUALIFIED
+%token T_NAME_QUALIFIED
+%token T_NAME_RELATIVE
+%token T_ATTRIBUTE
+%token T_ENUM

lib/PhpParser/Builder.php

@@ -2,11 +2,12 @@
 
 namespace PhpParser;
 
-interface Builder {
+interface Builder
+{
     /**
      * Returns the built node.
      *
      * @return Node The built node
      */
-    public function getNode(): Node;
+    public function getNode() : Node;
 }

lib/PhpParser/Builder/ClassConst.php

@@ -6,29 +6,27 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Const_;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Stmt;
 
-class ClassConst implements PhpParser\Builder {
+class ClassConst implements PhpParser\Builder
-    protected int $flags = 0;
+{
-    /** @var array<string, mixed> */
+    protected $flags = 0;
-    protected array $attributes = [];
+    protected $attributes = [];
-    /** @var list<Const_> */
+    protected $constants = [];
-    protected array $constants = [];
 
-    /** @var list<Node\AttributeGroup> */
+    /** @var Node\AttributeGroup[] */
-    protected array $attributeGroups = [];
+    protected $attributeGroups = [];
-    /** @var Identifier|Node\Name|Node\ComplexType|null */
+    /** @var Identifier|Node\Name|Node\ComplexType */
-    protected ?Node $type = null;
+    protected $type;
 
     /**
      * Creates a class constant builder
      *
-     * @param string|Identifier $name Name
+     * @param string|Identifier                          $name  Name
-     * @param Node\Expr|bool|null|int|float|string|array|\UnitEnum $value Value
+     * @param Node\Expr|bool|null|int|float|string|array $value Value
      */
     public function __construct($name, $value) {
         $this->constants = [new Const_($name, BuilderHelpers::normalizeValue($value))];
@@ -37,8 +35,8 @@ class ClassConst implements PhpParser\Builder {
     /**
      * Add another constant to const group
      *
-     * @param string|Identifier $name Name
+     * @param string|Identifier                          $name  Name
-     * @param Node\Expr|bool|null|int|float|string|array|\UnitEnum $value Value
+     * @param Node\Expr|bool|null|int|float|string|array $value Value
      *
      * @return $this The builder instance (for fluid interface)
      */
@@ -54,7 +52,7 @@ class ClassConst implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makePublic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PUBLIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PUBLIC);
 
         return $this;
     }
@@ -65,7 +63,7 @@ class ClassConst implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeProtected() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PROTECTED);
 
         return $this;
     }
@@ -76,7 +74,7 @@ class ClassConst implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makePrivate() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PRIVATE);
 
         return $this;
     }
@@ -87,7 +85,7 @@ class ClassConst implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeFinal() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::FINAL);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
 
         return $this;
     }

lib/PhpParser/Builder/Class_.php

@@ -4,27 +4,25 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
-class Class_ extends Declaration {
+class Class_ extends Declaration
-    protected string $name;
+{
-    protected ?Name $extends = null;
+    protected $name;
-    /** @var list<Name> */
+
-    protected array $implements = [];
+    protected $extends = null;
-    protected int $flags = 0;
+    protected $implements = [];
-    /** @var list<Stmt\TraitUse> */
+    protected $flags = 0;
-    protected array $uses = [];
+
-    /** @var list<Stmt\ClassConst> */
+    protected $uses = [];
-    protected array $constants = [];
+    protected $constants = [];
-    /** @var list<Stmt\Property> */
+    protected $properties = [];
-    protected array $properties = [];
+    protected $methods = [];
-    /** @var list<Stmt\ClassMethod> */
+
-    protected array $methods = [];
+    /** @var Node\AttributeGroup[] */
-    /** @var list<Node\AttributeGroup> */
+    protected $attributeGroups = [];
-    protected array $attributeGroups = [];
 
     /**
      * Creates a class builder.
@@ -69,7 +67,7 @@ class Class_ extends Declaration {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeAbstract() {
-        $this->flags = BuilderHelpers::addClassModifier($this->flags, Modifiers::ABSTRACT);
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Stmt\Class_::MODIFIER_ABSTRACT);
 
         return $this;
     }
@@ -80,18 +78,13 @@ class Class_ extends Declaration {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeFinal() {
-        $this->flags = BuilderHelpers::addClassModifier($this->flags, Modifiers::FINAL);
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
 
         return $this;
     }
 
-    /**
-     * Makes the class readonly.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
     public function makeReadonly() {
-        $this->flags = BuilderHelpers::addClassModifier($this->flags, Modifiers::READONLY);
+        $this->flags = BuilderHelpers::addClassModifier($this->flags, Stmt\Class_::MODIFIER_READONLY);
 
         return $this;
     }
@@ -106,18 +99,20 @@ class Class_ extends Declaration {
     public function addStmt($stmt) {
         $stmt = BuilderHelpers::normalizeNode($stmt);
 
-        if ($stmt instanceof Stmt\Property) {
+        $targets = [
-            $this->properties[] = $stmt;
+            Stmt\TraitUse::class    => &$this->uses,
-        } elseif ($stmt instanceof Stmt\ClassMethod) {
+            Stmt\ClassConst::class  => &$this->constants,
-            $this->methods[] = $stmt;
+            Stmt\Property::class    => &$this->properties,
-        } elseif ($stmt instanceof Stmt\TraitUse) {
+            Stmt\ClassMethod::class => &$this->methods,
-            $this->uses[] = $stmt;
+        ];
-        } elseif ($stmt instanceof Stmt\ClassConst) {
+
-            $this->constants[] = $stmt;
+        $class = \get_class($stmt);
-        } else {
+        if (!isset($targets[$class])) {
             throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
         }
 
+        $targets[$class][] = $stmt;
+
         return $this;
     }
 
@@ -139,7 +134,7 @@ class Class_ extends Declaration {
      *
      * @return Stmt\Class_ The built class node
      */
-    public function getNode(): PhpParser\Node {
+    public function getNode() : PhpParser\Node {
         return new Stmt\Class_($this->name, [
             'flags' => $this->flags,
             'extends' => $this->extends,

lib/PhpParser/Builder/Declaration.php

@@ -5,23 +5,16 @@ namespace PhpParser\Builder;
 use PhpParser;
 use PhpParser\BuilderHelpers;
 
-abstract class Declaration implements PhpParser\Builder {
+abstract class Declaration implements PhpParser\Builder
-    /** @var array<string, mixed> */
+{
-    protected array $attributes = [];
+    protected $attributes = [];
 
-    /**
-     * Adds a statement.
-     *
-     * @param PhpParser\Node\Stmt|PhpParser\Builder $stmt The statement to add
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
     abstract public function addStmt($stmt);
 
     /**
      * Adds multiple statements.
      *
-     * @param (PhpParser\Node\Stmt|PhpParser\Builder)[] $stmts The statements to add
+     * @param array $stmts The statements to add
      *
      * @return $this The builder instance (for fluid interface)
      */

lib/PhpParser/Builder/EnumCase.php

@@ -10,20 +10,19 @@ use PhpParser\Node;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Stmt;
 
-class EnumCase implements PhpParser\Builder {
+class EnumCase implements PhpParser\Builder
-    /** @var Identifier|string */
+{
     protected $name;
-    protected ?Node\Expr $value = null;
+    protected $value = null;
-    /** @var array<string, mixed> */
+    protected $attributes = [];
-    protected array $attributes = [];
 
-    /** @var list<Node\AttributeGroup> */
+    /** @var Node\AttributeGroup[] */
-    protected array $attributeGroups = [];
+    protected $attributeGroups = [];
 
     /**
      * Creates an enum case builder.
      *
-     * @param string|Identifier $name Name
+     * @param string|Identifier $name  Name
      */
     public function __construct($name) {
         $this->name = $name;

lib/PhpParser/Builder/Enum_.php

@@ -9,21 +9,20 @@ use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
-class Enum_ extends Declaration {
+class Enum_ extends Declaration
-    protected string $name;
+{
-    protected ?Identifier $scalarType = null;
+    protected $name;
-    /** @var list<Name> */
+    protected $scalarType = null;
-    protected array $implements = [];
+
-    /** @var list<Stmt\TraitUse> */
+    protected $implements = [];
-    protected array $uses = [];
+
-    /** @var list<Stmt\EnumCase> */
+    protected $uses = [];
-    protected array $enumCases = [];
+    protected $enumCases = [];
-    /** @var list<Stmt\ClassConst> */
+    protected $constants = [];
-    protected array $constants = [];
+    protected $methods = [];
-    /** @var list<Stmt\ClassMethod> */
+
-    protected array $methods = [];
+    /** @var Node\AttributeGroup[] */
-    /** @var list<Node\AttributeGroup> */
+    protected $attributeGroups = [];
-    protected array $attributeGroups = [];
 
     /**
      * Creates an enum builder.
@@ -37,7 +36,7 @@ class Enum_ extends Declaration {
     /**
      * Sets the scalar type.
      *
-     * @param string|Identifier $scalarType
+     * @param string|Identifier $type
      *
      * @return $this
      */
@@ -72,18 +71,20 @@ class Enum_ extends Declaration {
     public function addStmt($stmt) {
         $stmt = BuilderHelpers::normalizeNode($stmt);
 
-        if ($stmt instanceof Stmt\EnumCase) {
+        $targets = [
-            $this->enumCases[] = $stmt;
+            Stmt\TraitUse::class    => &$this->uses,
-        } elseif ($stmt instanceof Stmt\ClassMethod) {
+            Stmt\EnumCase::class    => &$this->enumCases,
-            $this->methods[] = $stmt;
+            Stmt\ClassConst::class  => &$this->constants,
-        } elseif ($stmt instanceof Stmt\TraitUse) {
+            Stmt\ClassMethod::class => &$this->methods,
-            $this->uses[] = $stmt;
+        ];
-        } elseif ($stmt instanceof Stmt\ClassConst) {
+
-            $this->constants[] = $stmt;
+        $class = \get_class($stmt);
-        } else {
+        if (!isset($targets[$class])) {
             throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
         }
 
+        $targets[$class][] = $stmt;
+
         return $this;
     }
 
@@ -105,7 +106,7 @@ class Enum_ extends Declaration {
      *
      * @return Stmt\Enum_ The built enum node
      */
-    public function getNode(): PhpParser\Node {
+    public function getNode() : PhpParser\Node {
         return new Stmt\Enum_($this->name, [
             'scalarType' => $this->scalarType,
             'implements' => $this->implements,

lib/PhpParser/Builder/FunctionLike.php

@@ -5,13 +5,13 @@ namespace PhpParser\Builder;
 use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 
-abstract class FunctionLike extends Declaration {
+abstract class FunctionLike extends Declaration
-    protected bool $returnByRef = false;
+{
-    /** @var Node\Param[] */
+    protected $returnByRef = false;
-    protected array $params = [];
+    protected $params = [];
 
-    /** @var Node\Identifier|Node\Name|Node\ComplexType|null */
+    /** @var string|Node\Name|Node\NullableType|null */
-    protected ?Node $returnType = null;
+    protected $returnType = null;
 
     /**
      * Make the function return by reference.
@@ -46,7 +46,7 @@ abstract class FunctionLike extends Declaration {
     /**
      * Adds multiple parameters.
      *
-     * @param (Node\Param|Param)[] $params The parameters to add
+     * @param array $params The parameters to add
      *
      * @return $this The builder instance (for fluid interface)
      */

lib/PhpParser/Builder/Function_.php

@@ -7,13 +7,13 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Function_ extends FunctionLike {
+class Function_ extends FunctionLike
-    protected string $name;
+{
-    /** @var list<Stmt> */
+    protected $name;
-    protected array $stmts = [];
+    protected $stmts = [];
 
-    /** @var list<Node\AttributeGroup> */
+    /** @var Node\AttributeGroup[] */
-    protected array $attributeGroups = [];
+    protected $attributeGroups = [];
 
     /**
      * Creates a function builder.
@@ -55,7 +55,7 @@ class Function_ extends FunctionLike {
      *
      * @return Stmt\Function_ The built function node
      */
-    public function getNode(): Node {
+    public function getNode() : Node {
         return new Stmt\Function_($this->name, [
             'byRef'      => $this->returnByRef,
             'params'     => $this->params,

lib/PhpParser/Builder/Interface_.php

@@ -8,16 +8,15 @@ use PhpParser\Node;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 
-class Interface_ extends Declaration {
+class Interface_ extends Declaration
-    protected string $name;
+{
-    /** @var list<Name> */
+    protected $name;
-    protected array $extends = [];
+    protected $extends = [];
-    /** @var list<Stmt\ClassConst> */
+    protected $constants = [];
-    protected array $constants = [];
+    protected $methods = [];
-    /** @var list<Stmt\ClassMethod> */
+
-    protected array $methods = [];
+    /** @var Node\AttributeGroup[] */
-    /** @var list<Node\AttributeGroup> */
+    protected $attributeGroups = [];
-    protected array $attributeGroups = [];
 
     /**
      * Creates an interface builder.
@@ -84,7 +83,7 @@ class Interface_ extends Declaration {
      *
      * @return Stmt\Interface_ The built interface node
      */
-    public function getNode(): PhpParser\Node {
+    public function getNode() : PhpParser\Node {
         return new Stmt\Interface_($this->name, [
             'extends' => $this->extends,
             'stmts' => array_merge($this->constants, $this->methods),

lib/PhpParser/Builder/Method.php

@@ -4,20 +4,19 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Method extends FunctionLike {
+class Method extends FunctionLike
-    protected string $name;
+{
+    protected $name;
+    protected $flags = 0;
 
-    protected int $flags = 0;
+    /** @var array|null */
+    protected $stmts = [];
 
-    /** @var list<Stmt>|null */
+    /** @var Node\AttributeGroup[] */
-    protected ?array $stmts = [];
+    protected $attributeGroups = [];
-
-    /** @var list<Node\AttributeGroup> */
-    protected array $attributeGroups = [];
 
     /**
      * Creates a method builder.
@@ -34,7 +33,7 @@ class Method extends FunctionLike {
      * @return $this The builder instance (for fluid interface)
      */
     public function makePublic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PUBLIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PUBLIC);
 
         return $this;
     }
@@ -45,7 +44,7 @@ class Method extends FunctionLike {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeProtected() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PROTECTED);
 
         return $this;
     }
@@ -56,7 +55,7 @@ class Method extends FunctionLike {
      * @return $this The builder instance (for fluid interface)
      */
     public function makePrivate() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PRIVATE);
 
         return $this;
     }
@@ -67,7 +66,7 @@ class Method extends FunctionLike {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeStatic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::STATIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_STATIC);
 
         return $this;
     }
@@ -82,7 +81,7 @@ class Method extends FunctionLike {
             throw new \LogicException('Cannot make method with statements abstract');
         }
 
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::ABSTRACT);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_ABSTRACT);
         $this->stmts = null; // abstract methods don't have statements
 
         return $this;
@@ -94,7 +93,7 @@ class Method extends FunctionLike {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeFinal() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::FINAL);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_FINAL);
 
         return $this;
     }
@@ -134,7 +133,7 @@ class Method extends FunctionLike {
      *
      * @return Stmt\ClassMethod The built method node
      */
-    public function getNode(): Node {
+    public function getNode() : Node {
         return new Stmt\ClassMethod($this->name, [
             'flags'      => $this->flags,
             'byRef'      => $this->returnByRef,

lib/PhpParser/Builder/Namespace_.php

@@ -7,10 +7,10 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Namespace_ extends Declaration {
+class Namespace_ extends Declaration
-    private ?Node\Name $name;
+{
-    /** @var Stmt[] */
+    private $name;
-    private array $stmts = [];
+    private $stmts = [];
 
     /**
      * Creates a namespace builder.
@@ -39,7 +39,7 @@ class Namespace_ extends Declaration {
      *
      * @return Stmt\Namespace_ The built node
      */
-    public function getNode(): Node {
+    public function getNode() : Node {
         return new Stmt\Namespace_($this->name, $this->stmts, $this->attributes);
     }
 }

lib/PhpParser/Builder/Param.php

@@ -4,19 +4,25 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Modifiers;
 use PhpParser\Node;
 
-class Param implements PhpParser\Builder {
+class Param implements PhpParser\Builder
-    protected string $name;
+{
-    protected ?Node\Expr $default = null;
+    protected $name;
-    /** @var Node\Identifier|Node\Name|Node\ComplexType|null */
+
-    protected ?Node $type = null;
+    protected $default = null;
-    protected bool $byRef = false;
+
-    protected int $flags = 0;
+    /** @var Node\Identifier|Node\Name|Node\NullableType|null */
-    protected bool $variadic = false;
+    protected $type = null;
-    /** @var list<Node\AttributeGroup> */
+
-    protected array $attributeGroups = [];
+    protected $byRef = false;
+
+    protected $variadic = false;
+
+    protected $flags = 0;
+
+    /** @var Node\AttributeGroup[] */
+    protected $attributeGroups = [];
 
     /**
      * Creates a parameter builder.
@@ -56,6 +62,19 @@ class Param implements PhpParser\Builder {
         return $this;
     }
 
+    /**
+     * Sets type for the parameter.
+     *
+     * @param string|Node\Name|Node\Identifier|Node\ComplexType $type Parameter type
+     *
+     * @return $this The builder instance (for fluid interface)
+     *
+     * @deprecated Use setType() instead
+     */
+    public function setTypeHint($type) {
+        return $this->setType($type);
+    }
+
     /**
      * Make the parameter accept the value by reference.
      *
@@ -84,7 +103,7 @@ class Param implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makePublic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PUBLIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Node\Stmt\Class_::MODIFIER_PUBLIC);
 
         return $this;
     }
@@ -95,7 +114,7 @@ class Param implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeProtected() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Node\Stmt\Class_::MODIFIER_PROTECTED);
 
         return $this;
     }
@@ -106,7 +125,7 @@ class Param implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makePrivate() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Node\Stmt\Class_::MODIFIER_PRIVATE);
 
         return $this;
     }
@@ -117,29 +136,7 @@ class Param implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeReadonly() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::READONLY);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Node\Stmt\Class_::MODIFIER_READONLY);
-
-        return $this;
-    }
-
-    /**
-     * Gives the promoted property private(set) visibility.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makePrivateSet() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE_SET);
-
-        return $this;
-    }
-
-    /**
-     * Gives the promoted property protected(set) visibility.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makeProtectedSet() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED_SET);
 
         return $this;
     }
@@ -162,7 +159,7 @@ class Param implements PhpParser\Builder {
      *
      * @return Node\Param The built parameter node
      */
-    public function getNode(): Node {
+    public function getNode() : Node {
         return new Node\Param(
             new Node\Expr\Variable($this->name),
             $this->default, $this->type, $this->byRef, $this->variadic, [], $this->flags, $this->attributeGroups

lib/PhpParser/Builder/Property.php

@@ -4,27 +4,25 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
-use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
 use PhpParser\Node\Stmt;
 use PhpParser\Node\ComplexType;
 
-class Property implements PhpParser\Builder {
+class Property implements PhpParser\Builder
-    protected string $name;
+{
+    protected $name;
 
-    protected int $flags = 0;
+    protected $flags = 0;
+    protected $default = null;
+    protected $attributes = [];
 
-    protected ?Node\Expr $default = null;
+    /** @var null|Identifier|Name|NullableType */
-    /** @var array<string, mixed> */
+    protected $type;
-    protected array $attributes = [];
+
-    /** @var null|Identifier|Name|ComplexType */
+    /** @var Node\AttributeGroup[] */
-    protected ?Node $type = null;
+    protected $attributeGroups = [];
-    /** @var list<Node\AttributeGroup> */
-    protected array $attributeGroups = [];
-    /** @var list<Node\PropertyHook> */
-    protected array $hooks = [];
 
     /**
      * Creates a property builder.
@@ -41,7 +39,7 @@ class Property implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makePublic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PUBLIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PUBLIC);
 
         return $this;
     }
@@ -52,7 +50,7 @@ class Property implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeProtected() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PROTECTED);
 
         return $this;
     }
@@ -63,7 +61,7 @@ class Property implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makePrivate() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_PRIVATE);
 
         return $this;
     }
@@ -74,7 +72,7 @@ class Property implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeStatic() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::STATIC);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_STATIC);
 
         return $this;
     }
@@ -85,51 +83,7 @@ class Property implements PhpParser\Builder {
      * @return $this The builder instance (for fluid interface)
      */
     public function makeReadonly() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::READONLY);
+        $this->flags = BuilderHelpers::addModifier($this->flags, Stmt\Class_::MODIFIER_READONLY);
-
-        return $this;
-    }
-
-    /**
-     * Makes the property abstract. Requires at least one property hook to be specified as well.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makeAbstract() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::ABSTRACT);
-
-        return $this;
-    }
-
-    /**
-     * Makes the property final.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makeFinal() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::FINAL);
-
-        return $this;
-    }
-
-    /**
-     * Gives the property private(set) visibility.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makePrivateSet() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PRIVATE_SET);
-
-        return $this;
-    }
-
-    /**
-     * Gives the property protected(set) visibility.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function makeProtectedSet() {
-        $this->flags = BuilderHelpers::addModifier($this->flags, Modifiers::PROTECTED_SET);
 
         return $this;
     }
@@ -188,36 +142,20 @@ class Property implements PhpParser\Builder {
         return $this;
     }
 
-    /**
-     * Adds a property hook.
-     *
-     * @return $this The builder instance (for fluid interface)
-     */
-    public function addHook(Node\PropertyHook $hook) {
-        $this->hooks[] = $hook;
-
-        return $this;
-    }
-
     /**
      * Returns the built class node.
      *
      * @return Stmt\Property The built property node
      */
-    public function getNode(): PhpParser\Node {
+    public function getNode() : PhpParser\Node {
-        if ($this->flags & Modifiers::ABSTRACT && !$this->hooks) {
-            throw new PhpParser\Error('Only hooked properties may be declared abstract');
-        }
-
         return new Stmt\Property(
-            $this->flags !== 0 ? $this->flags : Modifiers::PUBLIC,
+            $this->flags !== 0 ? $this->flags : Stmt\Class_::MODIFIER_PUBLIC,
             [
-                new Node\PropertyItem($this->name, $this->default)
+                new Stmt\PropertyProperty($this->name, $this->default)
             ],
             $this->attributes,
             $this->type,
-            $this->attributeGroups,
+            $this->attributeGroups
-            $this->hooks
         );
     }
 }

lib/PhpParser/Builder/TraitUse.php

@@ -7,11 +7,10 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class TraitUse implements Builder {
+class TraitUse implements Builder
-    /** @var Node\Name[] */
+{
-    protected array $traits = [];
+    protected $traits = [];
-    /** @var Stmt\TraitUseAdaptation[] */
+    protected $adaptations = [];
-    protected array $adaptations = [];
 
     /**
      * Creates a trait use builder.
@@ -59,7 +58,7 @@ class TraitUse implements Builder {
      *
      * @return Node The built node
      */
-    public function getNode(): Node {
+    public function getNode() : Node {
         return new Stmt\TraitUse($this->traits, $this->adaptations);
     }
 }

lib/PhpParser/Builder/TraitUseAdaptation.php

@@ -4,40 +4,43 @@ namespace PhpParser\Builder;
 
 use PhpParser\Builder;
 use PhpParser\BuilderHelpers;
-use PhpParser\Modifiers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class TraitUseAdaptation implements Builder {
+class TraitUseAdaptation implements Builder
-    private const TYPE_UNDEFINED  = 0;
+{
-    private const TYPE_ALIAS      = 1;
+    const TYPE_UNDEFINED  = 0;
-    private const TYPE_PRECEDENCE = 2;
+    const TYPE_ALIAS      = 1;
+    const TYPE_PRECEDENCE = 2;
 
-    protected int $type;
+    /** @var int Type of building adaptation */
-    protected ?Node\Name $trait;
+    protected $type;
-    protected Node\Identifier $method;
+
-    protected ?int $modifier = null;
+    protected $trait;
-    protected ?Node\Identifier $alias = null;
+    protected $method;
-    /** @var Node\Name[] */
+
-    protected array $insteadof = [];
+    protected $modifier = null;
+    protected $alias = null;
+
+    protected $insteadof = [];
 
     /**
      * Creates a trait use adaptation builder.
      *
-     * @param Node\Name|string|null $trait Name of adapted trait
+     * @param Node\Name|string|null  $trait  Name of adaptated trait
-     * @param Node\Identifier|string $method Name of adapted method
+     * @param Node\Identifier|string $method Name of adaptated method
      */
     public function __construct($trait, $method) {
         $this->type = self::TYPE_UNDEFINED;
 
-        $this->trait = is_null($trait) ? null : BuilderHelpers::normalizeName($trait);
+        $this->trait = is_null($trait)? null: BuilderHelpers::normalizeName($trait);
         $this->method = BuilderHelpers::normalizeIdentifier($method);
     }
 
     /**
      * Sets alias of method.
      *
-     * @param Node\Identifier|string $alias Alias for adapted method
+     * @param Node\Identifier|string $alias Alias for adaptated method
      *
      * @return $this The builder instance (for fluid interface)
      */
@@ -50,37 +53,37 @@ class TraitUseAdaptation implements Builder {
             throw new \LogicException('Cannot set alias for not alias adaptation buider');
         }
 
-        $this->alias = BuilderHelpers::normalizeIdentifier($alias);
+        $this->alias = $alias;
         return $this;
     }
 
     /**
-     * Sets adapted method public.
+     * Sets adaptated method public.
      *
      * @return $this The builder instance (for fluid interface)
      */
     public function makePublic() {
-        $this->setModifier(Modifiers::PUBLIC);
+        $this->setModifier(Stmt\Class_::MODIFIER_PUBLIC);
         return $this;
     }
 
     /**
-     * Sets adapted method protected.
+     * Sets adaptated method protected.
      *
      * @return $this The builder instance (for fluid interface)
      */
     public function makeProtected() {
-        $this->setModifier(Modifiers::PROTECTED);
+        $this->setModifier(Stmt\Class_::MODIFIER_PROTECTED);
         return $this;
     }
 
     /**
-     * Sets adapted method private.
+     * Sets adaptated method private.
      *
      * @return $this The builder instance (for fluid interface)
      */
     public function makePrivate() {
-        $this->setModifier(Modifiers::PRIVATE);
+        $this->setModifier(Stmt\Class_::MODIFIER_PRIVATE);
         return $this;
     }
 
@@ -111,7 +114,7 @@ class TraitUseAdaptation implements Builder {
         return $this;
     }
 
-    protected function setModifier(int $modifier): void {
+    protected function setModifier(int $modifier) {
         if ($this->type === self::TYPE_UNDEFINED) {
             $this->type = self::TYPE_ALIAS;
         }
@@ -132,7 +135,7 @@ class TraitUseAdaptation implements Builder {
      *
      * @return Node The built node
      */
-    public function getNode(): Node {
+    public function getNode() : Node {
         switch ($this->type) {
             case self::TYPE_ALIAS:
                 return new Stmt\TraitUseAdaptation\Alias($this->trait, $this->method, $this->modifier, $this->alias);

lib/PhpParser/Builder/Trait_.php

@@ -7,18 +7,15 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Trait_ extends Declaration {
+class Trait_ extends Declaration
-    protected string $name;
+{
-    /** @var list<Stmt\TraitUse> */
+    protected $name;
-    protected array $uses = [];
+    protected $uses = [];
-    /** @var list<Stmt\ClassConst> */
+    protected $properties = [];
-    protected array $constants = [];
+    protected $methods = [];
-    /** @var list<Stmt\Property> */
+
-    protected array $properties = [];
+    /** @var Node\AttributeGroup[] */
-    /** @var list<Stmt\ClassMethod> */
+    protected $attributeGroups = [];
-    protected array $methods = [];
-    /** @var list<Node\AttributeGroup> */
-    protected array $attributeGroups = [];
 
     /**
      * Creates an interface builder.
@@ -45,8 +42,6 @@ class Trait_ extends Declaration {
             $this->methods[] = $stmt;
         } elseif ($stmt instanceof Stmt\TraitUse) {
             $this->uses[] = $stmt;
-        } elseif ($stmt instanceof Stmt\ClassConst) {
-            $this->constants[] = $stmt;
         } else {
             throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
         }
@@ -72,10 +67,10 @@ class Trait_ extends Declaration {
      *
      * @return Stmt\Trait_ The built interface node
      */
-    public function getNode(): PhpParser\Node {
+    public function getNode() : PhpParser\Node {
         return new Stmt\Trait_(
             $this->name, [
-                'stmts' => array_merge($this->uses, $this->constants, $this->properties, $this->methods),
+                'stmts' => array_merge($this->uses, $this->properties, $this->methods),
                 'attrGroups' => $this->attributeGroups,
             ], $this->attributes
         );

lib/PhpParser/Builder/Use_.php

@@ -7,17 +7,17 @@ use PhpParser\BuilderHelpers;
 use PhpParser\Node;
 use PhpParser\Node\Stmt;
 
-class Use_ implements Builder {
+class Use_ implements Builder
-    protected Node\Name $name;
+{
-    /** @var Stmt\Use_::TYPE_* */
+    protected $name;
-    protected int $type;
+    protected $type;
-    protected ?string $alias = null;
+    protected $alias = null;
 
     /**
      * Creates a name use (alias) builder.
      *
      * @param Node\Name|string $name Name of the entity (namespace, class, function, constant) to alias
-     * @param Stmt\Use_::TYPE_* $type One of the Stmt\Use_::TYPE_* constants
+     * @param int              $type One of the Stmt\Use_::TYPE_* constants
      */
     public function __construct($name, int $type) {
         $this->name = BuilderHelpers::normalizeName($name);
@@ -41,9 +41,9 @@ class Use_ implements Builder {
      *
      * @return Stmt\Use_ The built node
      */
-    public function getNode(): Node {
+    public function getNode() : Node {
         return new Stmt\Use_([
-            new Node\UseItem($this->name, $this->alias)
+            new Stmt\UseUse($this->name, $this->alias)
         ], $this->type);
     }
 }

lib/PhpParser/BuilderFactory.php

@@ -10,14 +10,17 @@ use PhpParser\Node\Name;
 use PhpParser\Node\Scalar\String_;
 use PhpParser\Node\Stmt\Use_;
 
-class BuilderFactory {
+class BuilderFactory
+{
     /**
      * Creates an attribute node.
      *
      * @param string|Name $name Name of the attribute
-     * @param array $args Attribute named arguments
+     * @param array       $args Attribute named arguments
+     *
+     * @return Node\Attribute
      */
-    public function attribute($name, array $args = []): Node\Attribute {
+    public function attribute($name, array $args = []) : Node\Attribute {
         return new Node\Attribute(
             BuilderHelpers::normalizeName($name),
             $this->args($args)
@@ -31,7 +34,7 @@ class BuilderFactory {
      *
      * @return Builder\Namespace_ The created namespace builder
      */
-    public function namespace($name): Builder\Namespace_ {
+    public function namespace($name) : Builder\Namespace_ {
         return new Builder\Namespace_($name);
     }
 
@@ -42,7 +45,7 @@ class BuilderFactory {
      *
      * @return Builder\Class_ The created class builder
      */
-    public function class(string $name): Builder\Class_ {
+    public function class(string $name) : Builder\Class_ {
         return new Builder\Class_($name);
     }
 
@@ -53,7 +56,7 @@ class BuilderFactory {
      *
      * @return Builder\Interface_ The created interface builder
      */
-    public function interface(string $name): Builder\Interface_ {
+    public function interface(string $name) : Builder\Interface_ {
         return new Builder\Interface_($name);
     }
 
@@ -64,7 +67,7 @@ class BuilderFactory {
      *
      * @return Builder\Trait_ The created trait builder
      */
-    public function trait(string $name): Builder\Trait_ {
+    public function trait(string $name) : Builder\Trait_ {
         return new Builder\Trait_($name);
     }
 
@@ -75,7 +78,7 @@ class BuilderFactory {
      *
      * @return Builder\Enum_ The created enum builder
      */
-    public function enum(string $name): Builder\Enum_ {
+    public function enum(string $name) : Builder\Enum_ {
         return new Builder\Enum_($name);
     }
 
@@ -84,21 +87,21 @@ class BuilderFactory {
      *
      * @param Node\Name|string ...$traits Trait names
      *
-     * @return Builder\TraitUse The created trait use builder
+     * @return Builder\TraitUse The create trait use builder
      */
-    public function useTrait(...$traits): Builder\TraitUse {
+    public function useTrait(...$traits) : Builder\TraitUse {
         return new Builder\TraitUse(...$traits);
     }
 
     /**
      * Creates a trait use adaptation builder.
      *
-     * @param Node\Name|string|null $trait Trait name
+     * @param Node\Name|string|null  $trait  Trait name
      * @param Node\Identifier|string $method Method name
      *
-     * @return Builder\TraitUseAdaptation The created trait use adaptation builder
+     * @return Builder\TraitUseAdaptation The create trait use adaptation builder
      */
-    public function traitUseAdaptation($trait, $method = null): Builder\TraitUseAdaptation {
+    public function traitUseAdaptation($trait, $method = null) : Builder\TraitUseAdaptation {
         if ($method === null) {
             $method = $trait;
             $trait = null;
@@ -114,7 +117,7 @@ class BuilderFactory {
      *
      * @return Builder\Method The created method builder
      */
-    public function method(string $name): Builder\Method {
+    public function method(string $name) : Builder\Method {
         return new Builder\Method($name);
     }
 
@@ -125,7 +128,7 @@ class BuilderFactory {
      *
      * @return Builder\Param The created parameter builder
      */
-    public function param(string $name): Builder\Param {
+    public function param(string $name) : Builder\Param {
         return new Builder\Param($name);
     }
 
@@ -136,7 +139,7 @@ class BuilderFactory {
      *
      * @return Builder\Property The created property builder
      */
-    public function property(string $name): Builder\Property {
+    public function property(string $name) : Builder\Property {
         return new Builder\Property($name);
     }
 
@@ -147,7 +150,7 @@ class BuilderFactory {
      *
      * @return Builder\Function_ The created function builder
      */
-    public function function(string $name): Builder\Function_ {
+    public function function(string $name) : Builder\Function_ {
         return new Builder\Function_($name);
     }
 
@@ -158,7 +161,7 @@ class BuilderFactory {
      *
      * @return Builder\Use_ The created use builder
      */
-    public function use($name): Builder\Use_ {
+    public function use($name) : Builder\Use_ {
         return new Builder\Use_($name, Use_::TYPE_NORMAL);
     }
 
@@ -169,7 +172,7 @@ class BuilderFactory {
      *
      * @return Builder\Use_ The created use function builder
      */
-    public function useFunction($name): Builder\Use_ {
+    public function useFunction($name) : Builder\Use_ {
         return new Builder\Use_($name, Use_::TYPE_FUNCTION);
     }
 
@@ -180,39 +183,41 @@ class BuilderFactory {
      *
      * @return Builder\Use_ The created use const builder
      */
-    public function useConst($name): Builder\Use_ {
+    public function useConst($name) : Builder\Use_ {
         return new Builder\Use_($name, Use_::TYPE_CONSTANT);
     }
 
     /**
      * Creates a class constant builder.
      *
-     * @param string|Identifier $name Name
+     * @param string|Identifier                          $name  Name
      * @param Node\Expr|bool|null|int|float|string|array $value Value
      *
      * @return Builder\ClassConst The created use const builder
      */
-    public function classConst($name, $value): Builder\ClassConst {
+    public function classConst($name, $value) : Builder\ClassConst {
         return new Builder\ClassConst($name, $value);
     }
 
     /**
      * Creates an enum case builder.
      *
-     * @param string|Identifier $name Name
+     * @param string|Identifier $name  Name
      *
      * @return Builder\EnumCase The created use const builder
      */
-    public function enumCase($name): Builder\EnumCase {
+    public function enumCase($name) : Builder\EnumCase {
         return new Builder\EnumCase($name);
     }
 
     /**
      * Creates node a for a literal value.
      *
-     * @param Expr|bool|null|int|float|string|array|\UnitEnum $value $value
+     * @param Expr|bool|null|int|float|string|array $value $value
+     *
+     * @return Expr
      */
-    public function val($value): Expr {
+    public function val($value) : Expr {
         return BuilderHelpers::normalizeValue($value);
     }
 
@@ -220,8 +225,10 @@ class BuilderFactory {
      * Creates variable node.
      *
      * @param string|Expr $name Name
+     *
+     * @return Expr\Variable
      */
-    public function var($name): Expr\Variable {
+    public function var($name) : Expr\Variable {
         if (!\is_string($name) && !$name instanceof Expr) {
             throw new \LogicException('Variable name must be string or Expr');
         }
@@ -236,9 +243,9 @@ class BuilderFactory {
      *
      * @param array $args List of arguments to normalize
      *
-     * @return list<Arg>
+     * @return Arg[]
      */
-    public function args(array $args): array {
+    public function args(array $args) : array {
         $normalizedArgs = [];
         foreach ($args as $key => $arg) {
             if (!($arg instanceof Arg)) {
@@ -256,9 +263,11 @@ class BuilderFactory {
      * Creates a function call node.
      *
      * @param string|Name|Expr $name Function name
-     * @param array $args Function arguments
+     * @param array            $args Function arguments
+     *
+     * @return Expr\FuncCall
      */
-    public function funcCall($name, array $args = []): Expr\FuncCall {
+    public function funcCall($name, array $args = []) : Expr\FuncCall {
         return new Expr\FuncCall(
             BuilderHelpers::normalizeNameOrExpr($name),
             $this->args($args)
@@ -268,11 +277,13 @@ class BuilderFactory {
     /**
      * Creates a method call node.
      *
-     * @param Expr $var Variable the method is called on
+     * @param Expr                   $var  Variable the method is called on
      * @param string|Identifier|Expr $name Method name
-     * @param array $args Method arguments
+     * @param array                  $args Method arguments
+     *
+     * @return Expr\MethodCall
      */
-    public function methodCall(Expr $var, $name, array $args = []): Expr\MethodCall {
+    public function methodCall(Expr $var, $name, array $args = []) : Expr\MethodCall {
         return new Expr\MethodCall(
             $var,
             BuilderHelpers::normalizeIdentifierOrExpr($name),
@@ -283,11 +294,13 @@ class BuilderFactory {
     /**
      * Creates a static method call node.
      *
-     * @param string|Name|Expr $class Class name
+     * @param string|Name|Expr       $class Class name
-     * @param string|Identifier|Expr $name Method name
+     * @param string|Identifier|Expr $name  Method name
-     * @param array $args Method arguments
+     * @param array                  $args  Method arguments
+     *
+     * @return Expr\StaticCall
      */
-    public function staticCall($class, $name, array $args = []): Expr\StaticCall {
+    public function staticCall($class, $name, array $args = []) : Expr\StaticCall {
         return new Expr\StaticCall(
             BuilderHelpers::normalizeNameOrExpr($class),
             BuilderHelpers::normalizeIdentifierOrExpr($name),
@@ -299,9 +312,11 @@ class BuilderFactory {
      * Creates an object creation node.
      *
      * @param string|Name|Expr $class Class name
-     * @param array $args Constructor arguments
+     * @param array            $args  Constructor arguments
+     *
+     * @return Expr\New_
      */
-    public function new($class, array $args = []): Expr\New_ {
+    public function new($class, array $args = []) : Expr\New_ {
         return new Expr\New_(
             BuilderHelpers::normalizeNameOrExpr($class),
             $this->args($args)
@@ -312,18 +327,22 @@ class BuilderFactory {
      * Creates a constant fetch node.
      *
      * @param string|Name $name Constant name
+     *
+     * @return Expr\ConstFetch
      */
-    public function constFetch($name): Expr\ConstFetch {
+    public function constFetch($name) : Expr\ConstFetch {
         return new Expr\ConstFetch(BuilderHelpers::normalizeName($name));
     }
 
     /**
      * Creates a property fetch node.
      *
-     * @param Expr $var Variable holding object
+     * @param Expr                   $var  Variable holding object
      * @param string|Identifier|Expr $name Property name
+     *
+     * @return Expr\PropertyFetch
      */
-    public function propertyFetch(Expr $var, $name): Expr\PropertyFetch {
+    public function propertyFetch(Expr $var, $name) : Expr\PropertyFetch {
         return new Expr\PropertyFetch($var, BuilderHelpers::normalizeIdentifierOrExpr($name));
     }
 
@@ -331,7 +350,9 @@ class BuilderFactory {
      * Creates a class constant fetch node.
      *
      * @param string|Name|Expr $class Class name
-     * @param string|Identifier|Expr $name Constant name
+     * @param string|Identifier|Expr $name  Constant name
+     *
+     * @return Expr\ClassConstFetch
      */
     public function classConstFetch($class, $name): Expr\ClassConstFetch {
         return new Expr\ClassConstFetch(
@@ -344,8 +365,10 @@ class BuilderFactory {
      * Creates nested Concat nodes from a list of expressions.
      *
      * @param Expr|string ...$exprs Expressions or literal strings
+     *
+     * @return Concat
      */
-    public function concat(...$exprs): Concat {
+    public function concat(...$exprs) : Concat {
         $numExprs = count($exprs);
         if ($numExprs < 2) {
             throw new \LogicException('Expected at least two expressions');
@@ -360,8 +383,9 @@ class BuilderFactory {
 
     /**
      * @param string|Expr $expr
+     * @return Expr
      */
-    private function normalizeStringExpr($expr): Expr {
+    private function normalizeStringExpr($expr) : Expr {
         if ($expr instanceof Expr) {
             return $expr;
         }

lib/PhpParser/BuilderHelpers.php

@@ -6,7 +6,6 @@ use PhpParser\Node\ComplexType;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
-use PhpParser\Node\Name\FullyQualified;
 use PhpParser\Node\NullableType;
 use PhpParser\Node\Scalar;
 use PhpParser\Node\Stmt;
@@ -16,7 +15,8 @@ use PhpParser\Node\Stmt;
  *
  * @internal
  */
-final class BuilderHelpers {
+final class BuilderHelpers
+{
     /**
      * Normalizes a node: Converts builder objects to nodes.
      *
@@ -24,7 +24,7 @@ final class BuilderHelpers {
      *
      * @return Node The normalized node
      */
-    public static function normalizeNode($node): Node {
+    public static function normalizeNode($node) : Node {
         if ($node instanceof Builder) {
             return $node->getNode();
         }
@@ -45,7 +45,7 @@ final class BuilderHelpers {
      *
      * @return Stmt The normalized statement node
      */
-    public static function normalizeStmt($node): Stmt {
+    public static function normalizeStmt($node) : Stmt {
         $node = self::normalizeNode($node);
         if ($node instanceof Stmt) {
             return $node;
@@ -65,7 +65,7 @@ final class BuilderHelpers {
      *
      * @return Identifier The normalized identifier
      */
-    public static function normalizeIdentifier($name): Identifier {
+    public static function normalizeIdentifier($name) : Identifier {
         if ($name instanceof Identifier) {
             return $name;
         }
@@ -103,7 +103,7 @@ final class BuilderHelpers {
      *
      * @return Name The normalized name
      */
-    public static function normalizeName($name): Name {
+    public static function normalizeName($name) : Name {
         if ($name instanceof Name) {
             return $name;
         }
@@ -215,11 +215,11 @@ final class BuilderHelpers {
      * Normalizes a value: Converts nulls, booleans, integers,
      * floats, strings and arrays into their respective nodes
      *
-     * @param Node\Expr|bool|null|int|float|string|array|\UnitEnum $value The value to normalize
+     * @param Node\Expr|bool|null|int|float|string|array $value The value to normalize
      *
      * @return Expr The normalized value
      */
-    public static function normalizeValue($value): Expr {
+    public static function normalizeValue($value) : Expr {
         if ($value instanceof Node\Expr) {
             return $value;
         }
@@ -237,11 +237,11 @@ final class BuilderHelpers {
         }
 
         if (is_int($value)) {
-            return new Scalar\Int_($value);
+            return new Scalar\LNumber($value);
         }
 
         if (is_float($value)) {
-            return new Scalar\Float_($value);
+            return new Scalar\DNumber($value);
         }
 
         if (is_string($value)) {
@@ -254,12 +254,12 @@ final class BuilderHelpers {
             foreach ($value as $itemKey => $itemValue) {
                 // for consecutive, numeric keys don't generate keys
                 if (null !== $lastKey && ++$lastKey === $itemKey) {
-                    $items[] = new Node\ArrayItem(
+                    $items[] = new Expr\ArrayItem(
                         self::normalizeValue($itemValue)
                     );
                 } else {
                     $lastKey = null;
-                    $items[] = new Node\ArrayItem(
+                    $items[] = new Expr\ArrayItem(
                         self::normalizeValue($itemValue),
                         self::normalizeValue($itemKey)
                     );
@@ -269,10 +269,6 @@ final class BuilderHelpers {
             return new Expr\Array_($items);
         }
 
-        if ($value instanceof \UnitEnum) {
-            return new Expr\ClassConstFetch(new FullyQualified(\get_class($value)), new Identifier($value->name));
-        }
-
         throw new \LogicException('Invalid value');
     }
 
@@ -283,7 +279,7 @@ final class BuilderHelpers {
      *
      * @return Comment\Doc The normalized doc comment
      */
-    public static function normalizeDocComment($docComment): Comment\Doc {
+    public static function normalizeDocComment($docComment) : Comment\Doc {
         if ($docComment instanceof Comment\Doc) {
             return $docComment;
         }
@@ -302,7 +298,8 @@ final class BuilderHelpers {
      *
      * @return Node\AttributeGroup The Attribute Group
      */
-    public static function normalizeAttribute($attribute): Node\AttributeGroup {
+    public static function normalizeAttribute($attribute) : Node\AttributeGroup
+    {
         if ($attribute instanceof Node\AttributeGroup) {
             return $attribute;
         }
@@ -318,12 +315,12 @@ final class BuilderHelpers {
      * Adds a modifier and returns new modifier bitmask.
      *
      * @param int $modifiers Existing modifiers
-     * @param int $modifier Modifier to set
+     * @param int $modifier  Modifier to set
      *
      * @return int New modifiers
      */
-    public static function addModifier(int $modifiers, int $modifier): int {
+    public static function addModifier(int $modifiers, int $modifier) : int {
-        Modifiers::verifyModifier($modifiers, $modifier);
+        Stmt\Class_::verifyModifier($modifiers, $modifier);
         return $modifiers | $modifier;
     }
 
@@ -331,8 +328,8 @@ final class BuilderHelpers {
      * Adds a modifier and returns new modifier bitmask.
      * @return int New modifiers
      */
-    public static function addClassModifier(int $existingModifiers, int $modifierToSet): int {
+    public static function addClassModifier(int $existingModifiers, int $modifierToSet) : int {
-        Modifiers::verifyClassModifier($existingModifiers, $modifierToSet);
+        Stmt\Class_::verifyClassModifier($existingModifiers, $modifierToSet);
         return $existingModifiers | $modifierToSet;
     }
 }

lib/PhpParser/Comment.php

@@ -2,22 +2,23 @@
 
 namespace PhpParser;
 
-class Comment implements \JsonSerializable {
+class Comment implements \JsonSerializable
-    protected string $text;
+{
-    protected int $startLine;
+    protected $text;
-    protected int $startFilePos;
+    protected $startLine;
-    protected int $startTokenPos;
+    protected $startFilePos;
-    protected int $endLine;
+    protected $startTokenPos;
-    protected int $endFilePos;
+    protected $endLine;
-    protected int $endTokenPos;
+    protected $endFilePos;
+    protected $endTokenPos;
 
     /**
      * Constructs a comment node.
      *
-     * @param string $text Comment text (including comment delimiters like /*)
+     * @param string $text          Comment text (including comment delimiters like /*)
-     * @param int $startLine Line number the comment started on
+     * @param int    $startLine     Line number the comment started on
-     * @param int $startFilePos File offset the comment started on
+     * @param int    $startFilePos  File offset the comment started on
-     * @param int $startTokenPos Token offset the comment started on
+     * @param int    $startTokenPos Token offset the comment started on
      */
     public function __construct(
         string $text,
@@ -38,7 +39,7 @@ class Comment implements \JsonSerializable {
      *
      * @return string The comment text (including comment delimiters like /*)
      */
-    public function getText(): string {
+    public function getText() : string {
         return $this->text;
     }
 
@@ -46,9 +47,8 @@ class Comment implements \JsonSerializable {
      * Gets the line number the comment started on.
      *
      * @return int Line number (or -1 if not available)
-     * @phpstan-return -1|positive-int
      */
-    public function getStartLine(): int {
+    public function getStartLine() : int {
         return $this->startLine;
     }
 
@@ -57,7 +57,7 @@ class Comment implements \JsonSerializable {
      *
      * @return int File offset (or -1 if not available)
      */
-    public function getStartFilePos(): int {
+    public function getStartFilePos() : int {
         return $this->startFilePos;
     }
 
@@ -66,7 +66,7 @@ class Comment implements \JsonSerializable {
      *
      * @return int Token offset (or -1 if not available)
      */
-    public function getStartTokenPos(): int {
+    public function getStartTokenPos() : int {
         return $this->startTokenPos;
     }
 
@@ -74,9 +74,8 @@ class Comment implements \JsonSerializable {
      * Gets the line number the comment ends on.
      *
      * @return int Line number (or -1 if not available)
-     * @phpstan-return -1|positive-int
      */
-    public function getEndLine(): int {
+    public function getEndLine() : int {
         return $this->endLine;
     }
 
@@ -85,7 +84,7 @@ class Comment implements \JsonSerializable {
      *
      * @return int File offset (or -1 if not available)
      */
-    public function getEndFilePos(): int {
+    public function getEndFilePos() : int {
         return $this->endFilePos;
     }
 
@@ -94,16 +93,49 @@ class Comment implements \JsonSerializable {
      *
      * @return int Token offset (or -1 if not available)
      */
-    public function getEndTokenPos(): int {
+    public function getEndTokenPos() : int {
         return $this->endTokenPos;
     }
 
+    /**
+     * Gets the line number the comment started on.
+     *
+     * @deprecated Use getStartLine() instead
+     *
+     * @return int Line number
+     */
+    public function getLine() : int {
+        return $this->startLine;
+    }
+
+    /**
+     * Gets the file offset the comment started on.
+     *
+     * @deprecated Use getStartFilePos() instead
+     *
+     * @return int File offset
+     */
+    public function getFilePos() : int {
+        return $this->startFilePos;
+    }
+
+    /**
+     * Gets the token offset the comment started on.
+     *
+     * @deprecated Use getStartTokenPos() instead
+     *
+     * @return int Token offset
+     */
+    public function getTokenPos() : int {
+        return $this->startTokenPos;
+    }
+
     /**
      * Gets the comment text.
      *
      * @return string The comment text (including comment delimiters like /*)
      */
-    public function __toString(): string {
+    public function __toString() : string {
         return $this->text;
     }
 
@@ -112,19 +144,18 @@ class Comment implements \JsonSerializable {
      *
      * "Reformatted" here means that we try to clean up the whitespace at the
      * starts of the lines. This is necessary because we receive the comments
-     * without leading whitespace on the first line, but with leading whitespace
+     * without trailing whitespace on the first line, but with trailing whitespace
      * on all subsequent lines.
      *
-     * Additionally, this normalizes CRLF newlines to LF newlines.
+     * @return mixed|string
      */
-    public function getReformattedText(): string {
+    public function getReformattedText() {
-        $text = str_replace("\r\n", "\n", $this->text);
+        $text = trim($this->text);
         $newlinePos = strpos($text, "\n");
         if (false === $newlinePos) {
             // Single line comments don't need further processing
             return $text;
-        }
+        } elseif (preg_match('((*BSR_ANYCRLF)(*ANYCRLF)^.*(?:\R\s+\*.*)+$)', $text)) {
-        if (preg_match('(^.*(?:\n\s+\*.*)+$)', $text)) {
             // Multi line comment of the type
             //
             //     /*
@@ -133,9 +164,8 @@ class Comment implements \JsonSerializable {
             //      */
             //
             // is handled by replacing the whitespace sequences before the * by a single space
-            return preg_replace('(^\s+\*)m', ' *', $text);
+            return preg_replace('(^\s+\*)m', ' *', $this->text);
-        }
+        } elseif (preg_match('(^/\*\*?\s*[\r\n])', $text) && preg_match('(\n(\s*)\*/$)', $text, $matches)) {
-        if (preg_match('(^/\*\*?\s*\n)', $text) && preg_match('(\n(\s*)\*/$)', $text, $matches)) {
             // Multi line comment of the type
             //
             //    /*
@@ -147,8 +177,7 @@ class Comment implements \JsonSerializable {
             // */ on all lines. So if the last line is "    */", then "    " is removed at the
             // start of all lines.
             return preg_replace('(^' . preg_quote($matches[1]) . ')m', '', $text);
-        }
+        } elseif (preg_match('(^/\*\*?\s*(?!\s))', $text, $matches)) {
-        if (preg_match('(^/\*\*?\s*(?!\s))', $text, $matches)) {
             // Multi line comment of the type
             //
             //     /* Some text.
@@ -175,9 +204,9 @@ class Comment implements \JsonSerializable {
      * @param string $str String to check
      * @return int Length in characters. Tabs count as single characters.
      */
-    private function getShortestWhitespacePrefixLen(string $str): int {
+    private function getShortestWhitespacePrefixLen(string $str) : int {
         $lines = explode("\n", $str);
-        $shortestPrefixLen = \PHP_INT_MAX;
+        $shortestPrefixLen = \INF;
         foreach ($lines as $line) {
             preg_match('(^\s*)', $line, $matches);
             $prefixLen = strlen($matches[0]);
@@ -189,9 +218,10 @@ class Comment implements \JsonSerializable {
     }
 
     /**
-     * @return array{nodeType:string, text:mixed, line:mixed, filePos:mixed}
+     * @return       array
+     * @psalm-return array{nodeType:string, text:mixed, line:mixed, filePos:mixed}
      */
-    public function jsonSerialize(): array {
+    public function jsonSerialize() : array {
         // Technically not a node, but we make it look like one anyway
         $type = $this instanceof Comment\Doc ? 'Comment_Doc' : 'Comment';
         return [

lib/PhpParser/Comment/Doc.php

@@ -2,5 +2,6 @@
 
 namespace PhpParser\Comment;
 
-class Doc extends \PhpParser\Comment {
+class Doc extends \PhpParser\Comment
+{
 }

lib/PhpParser/ConstExprEvaluationException.php

@@ -1,6 +1,6 @@
-<?php declare(strict_types=1);
+<?php
 
 namespace PhpParser;
 
-class ConstExprEvaluationException extends \Exception {
+class ConstExprEvaluationException extends \Exception
-}
+{}

lib/PhpParser/ConstExprEvaluator.php

@@ -1,12 +1,11 @@
-<?php declare(strict_types=1);
+<?php
 
 namespace PhpParser;
 
+use function array_merge;
 use PhpParser\Node\Expr;
 use PhpParser\Node\Scalar;
 
-use function array_merge;
-
 /**
  * Evaluates constant expressions.
  *
@@ -26,8 +25,8 @@ use function array_merge;
  * point to string conversions are affected by the precision ini setting. Secondly, they are also
  * affected by the LC_NUMERIC locale.
  */
-class ConstExprEvaluator {
+class ConstExprEvaluator
-    /** @var callable|null */
+{
     private $fallbackEvaluator;
 
     /**
@@ -39,7 +38,7 @@ class ConstExprEvaluator {
      * @param callable|null $fallbackEvaluator To call if subexpression cannot be evaluated
      */
     public function __construct(?callable $fallbackEvaluator = null) {
-        $this->fallbackEvaluator = $fallbackEvaluator ?? function (Expr $expr) {
+        $this->fallbackEvaluator = $fallbackEvaluator ?? function(Expr $expr) {
             throw new ConstExprEvaluationException(
                 "Expression of type {$expr->getType()} cannot be evaluated"
             );
@@ -64,7 +63,7 @@ class ConstExprEvaluator {
      * @throws ConstExprEvaluationException if the expression cannot be evaluated or an error occurred
      */
     public function evaluateSilently(Expr $expr) {
-        set_error_handler(function ($num, $str, $file, $line) {
+        set_error_handler(function($num, $str, $file, $line) {
             throw new \ErrorException($str, 0, $num, $file, $line);
         });
 
@@ -102,10 +101,9 @@ class ConstExprEvaluator {
         return $this->evaluate($expr);
     }
 
-    /** @return mixed */
     private function evaluate(Expr $expr) {
-        if ($expr instanceof Scalar\Int_
+        if ($expr instanceof Scalar\LNumber
-            || $expr instanceof Scalar\Float_
+            || $expr instanceof Scalar\DNumber
             || $expr instanceof Scalar\String_
         ) {
             return $expr->value;
@@ -148,7 +146,7 @@ class ConstExprEvaluator {
         return ($this->fallbackEvaluator)($expr);
     }
 
-    private function evaluateArray(Expr\Array_ $expr): array {
+    private function evaluateArray(Expr\Array_ $expr) {
         $array = [];
         foreach ($expr->items as $item) {
             if (null !== $item->key) {
@@ -162,7 +160,6 @@ class ConstExprEvaluator {
         return $array;
     }
 
-    /** @return mixed */
     private function evaluateTernary(Expr\Ternary $expr) {
         if (null === $expr->if) {
             return $this->evaluate($expr->cond) ?: $this->evaluate($expr->else);
@@ -173,7 +170,6 @@ class ConstExprEvaluator {
             : $this->evaluate($expr->else);
     }
 
-    /** @return mixed */
     private function evaluateBinaryOp(Expr\BinaryOp $expr) {
         if ($expr instanceof Expr\BinaryOp\Coalesce
             && $expr->left instanceof Expr\ArrayDimFetch
@@ -220,7 +216,6 @@ class ConstExprEvaluator {
         throw new \Exception('Should not happen');
     }
 
-    /** @return mixed */
     private function evaluateConstFetch(Expr\ConstFetch $expr) {
         $name = $expr->name->toLowerString();
         switch ($name) {

lib/PhpParser/Error.php

@@ -2,20 +2,25 @@
 
 namespace PhpParser;
 
-class Error extends \RuntimeException {
+class Error extends \RuntimeException
-    protected string $rawMessage;
+{
-    /** @var array<string, mixed> */
+    protected $rawMessage;
-    protected array $attributes;
+    protected $attributes;
 
     /**
      * Creates an Exception signifying a parse error.
      *
-     * @param string $message Error message
+     * @param string    $message    Error message
-     * @param array<string, mixed> $attributes Attributes of node/token where error occurred
+     * @param array|int $attributes Attributes of node/token where error occurred
+     *                              (or start line of error -- deprecated)
      */
-    public function __construct(string $message, array $attributes = []) {
+    public function __construct(string $message, $attributes = []) {
         $this->rawMessage = $message;
-        $this->attributes = $attributes;
+        if (is_array($attributes)) {
+            $this->attributes = $attributes;
+        } else {
+            $this->attributes = ['startLine' => $attributes];
+        }
         $this->updateMessage();
     }
 
@@ -24,7 +29,7 @@ class Error extends \RuntimeException {
      *
      * @return string Error message
      */
-    public function getRawMessage(): string {
+    public function getRawMessage() : string {
         return $this->rawMessage;
     }
 
@@ -32,9 +37,8 @@ class Error extends \RuntimeException {
      * Gets the line the error starts in.
      *
      * @return int Error start line
-     * @phpstan-return -1|positive-int
      */
-    public function getStartLine(): int {
+    public function getStartLine() : int {
         return $this->attributes['startLine'] ?? -1;
     }
 
@@ -42,27 +46,26 @@ class Error extends \RuntimeException {
      * Gets the line the error ends in.
      *
      * @return int Error end line
-     * @phpstan-return -1|positive-int
      */
-    public function getEndLine(): int {
+    public function getEndLine() : int {
         return $this->attributes['endLine'] ?? -1;
     }
 
     /**
      * Gets the attributes of the node/token the error occurred at.
      *
-     * @return array<string, mixed>
+     * @return array
      */
-    public function getAttributes(): array {
+    public function getAttributes() : array {
         return $this->attributes;
     }
 
     /**
      * Sets the attributes of the node/token the error occurred at.
      *
-     * @param array<string, mixed> $attributes
+     * @param array $attributes
      */
-    public function setAttributes(array $attributes): void {
+    public function setAttributes(array $attributes) {
         $this->attributes = $attributes;
         $this->updateMessage();
     }
@@ -72,7 +75,7 @@ class Error extends \RuntimeException {
      *
      * @param string $message Error message
      */
-    public function setRawMessage(string $message): void {
+    public function setRawMessage(string $message) {
         $this->rawMessage = $message;
         $this->updateMessage();
     }
@@ -82,7 +85,7 @@ class Error extends \RuntimeException {
      *
      * @param int $line Error start line
      */
-    public function setStartLine(int $line): void {
+    public function setStartLine(int $line) {
         $this->attributes['startLine'] = $line;
         $this->updateMessage();
     }
@@ -91,8 +94,10 @@ class Error extends \RuntimeException {
      * Returns whether the error has start and end column information.
      *
      * For column information enable the startFilePos and endFilePos in the lexer options.
+     *
+     * @return bool
      */
-    public function hasColumnInfo(): bool {
+    public function hasColumnInfo() : bool {
         return isset($this->attributes['startFilePos'], $this->attributes['endFilePos']);
     }
 
@@ -100,8 +105,9 @@ class Error extends \RuntimeException {
      * Gets the start column (1-based) into the line where the error started.
      *
      * @param string $code Source code of the file
+     * @return int
      */
-    public function getStartColumn(string $code): int {
+    public function getStartColumn(string $code) : int {
         if (!$this->hasColumnInfo()) {
             throw new \RuntimeException('Error does not have column information');
         }
@@ -113,8 +119,9 @@ class Error extends \RuntimeException {
      * Gets the end column (1-based) into the line where the error ended.
      *
      * @param string $code Source code of the file
+     * @return int
      */
-    public function getEndColumn(string $code): int {
+    public function getEndColumn(string $code) : int {
         if (!$this->hasColumnInfo()) {
             throw new \RuntimeException('Error does not have column information');
         }
@@ -129,7 +136,7 @@ class Error extends \RuntimeException {
      *
      * @return string Formatted message
      */
-    public function getMessageWithColumnInfo(string $code): string {
+    public function getMessageWithColumnInfo(string $code) : string {
         return sprintf(
             '%s from %d:%d to %d:%d', $this->getRawMessage(),
             $this->getStartLine(), $this->getStartColumn($code),
@@ -141,11 +148,11 @@ class Error extends \RuntimeException {
      * Converts a file offset into a column.
      *
      * @param string $code Source code that $pos indexes into
-     * @param int $pos 0-based position in $code
+     * @param int    $pos  0-based position in $code
      *
      * @return int 1-based column (relative to start of line)
      */
-    private function toColumn(string $code, int $pos): int {
+    private function toColumn(string $code, int $pos) : int {
         if ($pos > strlen($code)) {
             throw new \RuntimeException('Invalid position information');
         }
@@ -161,7 +168,7 @@ class Error extends \RuntimeException {
     /**
      * Updates the exception message after a change to rawMessage or rawLine.
      */
-    protected function updateMessage(): void {
+    protected function updateMessage() {
         $this->message = $this->rawMessage;
 
         if (-1 === $this->getStartLine()) {

lib/PhpParser/ErrorHandler.php

@@ -2,11 +2,12 @@
 
 namespace PhpParser;
 
-interface ErrorHandler {
+interface ErrorHandler
+{
     /**
      * Handle an error generated during lexing, parsing or some other operation.
      *
      * @param Error $error The error that needs to be handled
      */
-    public function handleError(Error $error): void;
+    public function handleError(Error $error);
 }

lib/PhpParser/ErrorHandler/Collecting.php

@@ -10,11 +10,12 @@ use PhpParser\ErrorHandler;
  *
  * This allows graceful handling of errors.
  */
-class Collecting implements ErrorHandler {
+class Collecting implements ErrorHandler
+{
     /** @var Error[] Collected errors */
-    private array $errors = [];
+    private $errors = [];
 
-    public function handleError(Error $error): void {
+    public function handleError(Error $error) {
         $this->errors[] = $error;
     }
 
@@ -23,21 +24,23 @@ class Collecting implements ErrorHandler {
      *
      * @return Error[]
      */
-    public function getErrors(): array {
+    public function getErrors() : array {
         return $this->errors;
     }
 
     /**
      * Check whether there are any errors.
+     *
+     * @return bool
      */
-    public function hasErrors(): bool {
+    public function hasErrors() : bool {
         return !empty($this->errors);
     }
 
     /**
      * Reset/clear collected errors.
      */
-    public function clearErrors(): void {
+    public function clearErrors() {
         $this->errors = [];
     }
 }

lib/PhpParser/ErrorHandler/Throwing.php

@@ -10,8 +10,9 @@ use PhpParser\ErrorHandler;
  *
  * This is the default strategy used by all components.
  */
-class Throwing implements ErrorHandler {
+class Throwing implements ErrorHandler
-    public function handleError(Error $error): void {
+{
+    public function handleError(Error $error) {
         throw $error;
     }
 }

lib/PhpParser/Internal/DiffElem.php

@@ -5,24 +5,20 @@ namespace PhpParser\Internal;
 /**
  * @internal
  */
-class DiffElem {
+class DiffElem
-    public const TYPE_KEEP = 0;
+{
-    public const TYPE_REMOVE = 1;
+    const TYPE_KEEP = 0;
-    public const TYPE_ADD = 2;
+    const TYPE_REMOVE = 1;
-    public const TYPE_REPLACE = 3;
+    const TYPE_ADD = 2;
+    const TYPE_REPLACE = 3;
 
     /** @var int One of the TYPE_* constants */
-    public int $type;
+    public $type;
     /** @var mixed Is null for add operations */
     public $old;
     /** @var mixed Is null for remove operations */
     public $new;
 
-    /**
-     * @param int $type One of the TYPE_* constants
-     * @param mixed $old Is null for add operations
-     * @param mixed $new Is null for remove operations
-     */
     public function __construct(int $type, $old, $new) {
         $this->type = $type;
         $this->old = $old;

lib/PhpParser/Internal/Differ.php

@@ -8,17 +8,16 @@ namespace PhpParser\Internal;
  * Myers, Eugene W. "An O (ND) difference algorithm and its variations."
  * Algorithmica 1.1 (1986): 251-266.
  *
- * @template T
  * @internal
  */
-class Differ {
+class Differ
-    /** @var callable(T, T): bool */
+{
     private $isEqual;
 
     /**
      * Create differ over the given equality relation.
      *
-     * @param callable(T, T): bool $isEqual Equality relation
+     * @param callable $isEqual Equality relation with signature function($a, $b) : bool
      */
     public function __construct(callable $isEqual) {
         $this->isEqual = $isEqual;
@@ -27,14 +26,12 @@ class Differ {
     /**
      * Calculate diff (edit script) from $old to $new.
      *
-     * @param T[] $old Original array
+     * @param array $old Original array
-     * @param T[] $new New array
+     * @param array $new New array
      *
      * @return DiffElem[] Diff (edit script)
      */
-    public function diff(array $old, array $new): array {
+    public function diff(array $old, array $new) {
-        $old = \array_values($old);
-        $new = \array_values($new);
         list($trace, $x, $y) = $this->calculateTrace($old, $new);
         return $this->extractDiff($trace, $x, $y, $old, $new);
     }
@@ -45,37 +42,32 @@ class Differ {
      * If a sequence of remove operations is followed by the same number of add operations, these
      * will be coalesced into replace operations.
      *
-     * @param T[] $old Original array
+     * @param array $old Original array
-     * @param T[] $new New array
+     * @param array $new New array
      *
      * @return DiffElem[] Diff (edit script), including replace operations
      */
-    public function diffWithReplacements(array $old, array $new): array {
+    public function diffWithReplacements(array $old, array $new) {
         return $this->coalesceReplacements($this->diff($old, $new));
     }
 
-    /**
+    private function calculateTrace(array $a, array $b) {
-     * @param T[] $old
+        $n = \count($a);
-     * @param T[] $new
+        $m = \count($b);
-     * @return array{array<int, array<int, int>>, int, int}
-     */
-    private function calculateTrace(array $old, array $new): array {
-        $n = \count($old);
-        $m = \count($new);
         $max = $n + $m;
         $v = [1 => 0];
         $trace = [];
         for ($d = 0; $d <= $max; $d++) {
             $trace[] = $v;
             for ($k = -$d; $k <= $d; $k += 2) {
-                if ($k === -$d || ($k !== $d && $v[$k - 1] < $v[$k + 1])) {
+                if ($k === -$d || ($k !== $d && $v[$k-1] < $v[$k+1])) {
-                    $x = $v[$k + 1];
+                    $x = $v[$k+1];
                 } else {
-                    $x = $v[$k - 1] + 1;
+                    $x = $v[$k-1] + 1;
                 }
 
                 $y = $x - $k;
-                while ($x < $n && $y < $m && ($this->isEqual)($old[$x], $new[$y])) {
+                while ($x < $n && $y < $m && ($this->isEqual)($a[$x], $b[$y])) {
                     $x++;
                     $y++;
                 }
@@ -89,19 +81,13 @@ class Differ {
         throw new \Exception('Should not happen');
     }
 
-    /**
+    private function extractDiff(array $trace, int $x, int $y, array $a, array $b) {
-     * @param array<int, array<int, int>> $trace
-     * @param T[] $old
-     * @param T[] $new
-     * @return DiffElem[]
-     */
-    private function extractDiff(array $trace, int $x, int $y, array $old, array $new): array {
         $result = [];
         for ($d = \count($trace) - 1; $d >= 0; $d--) {
             $v = $trace[$d];
             $k = $x - $y;
 
-            if ($k === -$d || ($k !== $d && $v[$k - 1] < $v[$k + 1])) {
+            if ($k === -$d || ($k !== $d && $v[$k-1] < $v[$k+1])) {
                 $prevK = $k + 1;
             } else {
                 $prevK = $k - 1;
@@ -111,7 +97,7 @@ class Differ {
             $prevY = $prevX - $prevK;
 
             while ($x > $prevX && $y > $prevY) {
-                $result[] = new DiffElem(DiffElem::TYPE_KEEP, $old[$x - 1], $new[$y - 1]);
+                $result[] = new DiffElem(DiffElem::TYPE_KEEP, $a[$x-1], $b[$y-1]);
                 $x--;
                 $y--;
             }
@@ -121,12 +107,12 @@ class Differ {
             }
 
             while ($x > $prevX) {
-                $result[] = new DiffElem(DiffElem::TYPE_REMOVE, $old[$x - 1], null);
+                $result[] = new DiffElem(DiffElem::TYPE_REMOVE, $a[$x-1], null);
                 $x--;
             }
 
             while ($y > $prevY) {
-                $result[] = new DiffElem(DiffElem::TYPE_ADD, null, $new[$y - 1]);
+                $result[] = new DiffElem(DiffElem::TYPE_ADD, null, $b[$y-1]);
                 $y--;
             }
         }
@@ -139,7 +125,7 @@ class Differ {
      * @param DiffElem[] $diff
      * @return DiffElem[]
      */
-    private function coalesceReplacements(array $diff): array {
+    private function coalesceReplacements(array $diff) {
         $newDiff = [];
         $c = \count($diff);
         for ($i = 0; $i < $c; $i++) {

lib/PhpParser/Internal/PrintableNewAnonClassNode.php

@@ -15,28 +15,21 @@ use PhpParser\Node\Expr;
  *
  * @internal
  */
-class PrintableNewAnonClassNode extends Expr {
+class PrintableNewAnonClassNode extends Expr
+{
     /** @var Node\AttributeGroup[] PHP attribute groups */
-    public array $attrGroups;
+    public $attrGroups;
     /** @var int Modifiers */
-    public int $flags;
+    public $flags;
-    /** @var (Node\Arg|Node\VariadicPlaceholder)[] Arguments */
+    /** @var Node\Arg[] Arguments */
-    public array $args;
+    public $args;
     /** @var null|Node\Name Name of extended class */
-    public ?Node\Name $extends;
+    public $extends;
     /** @var Node\Name[] Names of implemented interfaces */
-    public array $implements;
+    public $implements;
     /** @var Node\Stmt[] Statements */
-    public array $stmts;
+    public $stmts;
 
-    /**
-     * @param Node\AttributeGroup[] $attrGroups PHP attribute groups
-     * @param (Node\Arg|Node\VariadicPlaceholder)[] $args Arguments
-     * @param Node\Name|null $extends Name of extended class
-     * @param Node\Name[] $implements Names of implemented interfaces
-     * @param Node\Stmt[] $stmts Statements
-     * @param array<string, mixed> $attributes Attributes
-     */
     public function __construct(
         array $attrGroups, int $flags, array $args, ?Node\Name $extends, array $implements,
         array $stmts, array $attributes
@@ -50,7 +43,7 @@ class PrintableNewAnonClassNode extends Expr {
         $this->stmts = $stmts;
     }
 
-    public static function fromNewNode(Expr\New_ $newNode): self {
+    public static function fromNewNode(Expr\New_ $newNode) {
         $class = $newNode->class;
         assert($class instanceof Node\Stmt\Class_);
         // We don't assert that $class->name is null here, to allow consumers to assign unique names
@@ -61,11 +54,11 @@ class PrintableNewAnonClassNode extends Expr {
         );
     }
 
-    public function getType(): string {
+    public function getType() : string {
         return 'Expr_PrintableNewAnonClass';
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['attrGroups', 'flags', 'args', 'extends', 'implements', 'stmts'];
     }
 }

lib/PhpParser/Internal/TokenPolyfill.php

@@ -1,237 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Internal;
-
-if (\PHP_VERSION_ID >= 80000) {
-    class TokenPolyfill extends \PhpToken {
-    }
-    return;
-}
-
-/**
- * This is a polyfill for the PhpToken class introduced in PHP 8.0. We do not actually polyfill
- * PhpToken, because composer might end up picking a different polyfill implementation, which does
- * not meet our requirements.
- *
- * @internal
- */
-class TokenPolyfill {
-    /** @var int The ID of the token. Either a T_* constant of a character code < 256. */
-    public int $id;
-    /** @var string The textual content of the token. */
-    public string $text;
-    /** @var int The 1-based starting line of the token (or -1 if unknown). */
-    public int $line;
-    /** @var int The 0-based starting position of the token (or -1 if unknown). */
-    public int $pos;
-
-    /** @var array<int, bool> Tokens ignored by the PHP parser. */
-    private const IGNORABLE_TOKENS = [
-        \T_WHITESPACE => true,
-        \T_COMMENT => true,
-        \T_DOC_COMMENT => true,
-        \T_OPEN_TAG => true,
-    ];
-
-    /** @var array<int, bool> Tokens that may be part of a T_NAME_* identifier. */
-    private static array $identifierTokens;
-
-    /**
-     * Create a Token with the given ID and text, as well optional line and position information.
-     */
-    final public function __construct(int $id, string $text, int $line = -1, int $pos = -1) {
-        $this->id = $id;
-        $this->text = $text;
-        $this->line = $line;
-        $this->pos = $pos;
-    }
-
-    /**
-     * Get the name of the token. For single-char tokens this will be the token character.
-     * Otherwise it will be a T_* style name, or null if the token ID is unknown.
-     */
-    public function getTokenName(): ?string {
-        if ($this->id < 256) {
-            return \chr($this->id);
-        }
-
-        $name = token_name($this->id);
-        return $name === 'UNKNOWN' ? null : $name;
-    }
-
-    /**
-     * Check whether the token is of the given kind. The kind may be either an integer that matches
-     * the token ID, a string that matches the token text, or an array of integers/strings. In the
-     * latter case, the function returns true if any of the kinds in the array match.
-     *
-     * @param int|string|(int|string)[] $kind
-     */
-    public function is($kind): bool {
-        if (\is_int($kind)) {
-            return $this->id === $kind;
-        }
-        if (\is_string($kind)) {
-            return $this->text === $kind;
-        }
-        if (\is_array($kind)) {
-            foreach ($kind as $entry) {
-                if (\is_int($entry)) {
-                    if ($this->id === $entry) {
-                        return true;
-                    }
-                } elseif (\is_string($entry)) {
-                    if ($this->text === $entry) {
-                        return true;
-                    }
-                } else {
-                    throw new \TypeError(
-                        'Argument #1 ($kind) must only have elements of type string|int, ' .
-                        gettype($entry) . ' given');
-                }
-            }
-            return false;
-        }
-        throw new \TypeError(
-            'Argument #1 ($kind) must be of type string|int|array, ' .gettype($kind) . ' given');
-    }
-
-    /**
-     * Check whether this token would be ignored by the PHP parser. Returns true for T_WHITESPACE,
-     * T_COMMENT, T_DOC_COMMENT and T_OPEN_TAG, and false for everything else.
-     */
-    public function isIgnorable(): bool {
-        return isset(self::IGNORABLE_TOKENS[$this->id]);
-    }
-
-    /**
-     * Return the textual content of the token.
-     */
-    public function __toString(): string {
-        return $this->text;
-    }
-
-    /**
-     * Tokenize the given source code and return an array of tokens.
-     *
-     * This performs certain canonicalizations to match the PHP 8.0 token format:
-     *  * Bad characters are represented using T_BAD_CHARACTER rather than omitted.
-     *  * T_COMMENT does not include trailing newlines, instead the newline is part of a following
-     *    T_WHITESPACE token.
-     *  * Namespaced names are represented using T_NAME_* tokens.
-     *
-     * @return static[]
-     */
-    public static function tokenize(string $code, int $flags = 0): array {
-        self::init();
-
-        $tokens = [];
-        $line = 1;
-        $pos = 0;
-        $origTokens = \token_get_all($code, $flags);
-
-        $numTokens = \count($origTokens);
-        for ($i = 0; $i < $numTokens; $i++) {
-            $token = $origTokens[$i];
-            if (\is_string($token)) {
-                if (\strlen($token) === 2) {
-                    // b" and B" are tokenized as single-char tokens, even though they aren't.
-                    $tokens[] = new static(\ord('"'), $token, $line, $pos);
-                    $pos += 2;
-                } else {
-                    $tokens[] = new static(\ord($token), $token, $line, $pos);
-                    $pos++;
-                }
-            } else {
-                $id = $token[0];
-                $text = $token[1];
-
-                // Emulate PHP 8.0 comment format, which does not include trailing whitespace anymore.
-                if ($id === \T_COMMENT && \substr($text, 0, 2) !== '/*' &&
-                    \preg_match('/(\r\n|\n|\r)$/D', $text, $matches)
-                ) {
-                    $trailingNewline = $matches[0];
-                    $text = \substr($text, 0, -\strlen($trailingNewline));
-                    $tokens[] = new static($id, $text, $line, $pos);
-                    $pos += \strlen($text);
-
-                    if ($i + 1 < $numTokens && $origTokens[$i + 1][0] === \T_WHITESPACE) {
-                        // Move trailing newline into following T_WHITESPACE token, if it already exists.
-                        $origTokens[$i + 1][1] = $trailingNewline . $origTokens[$i + 1][1];
-                        $origTokens[$i + 1][2]--;
-                    } else {
-                        // Otherwise, we need to create a new T_WHITESPACE token.
-                        $tokens[] = new static(\T_WHITESPACE, $trailingNewline, $line, $pos);
-                        $line++;
-                        $pos += \strlen($trailingNewline);
-                    }
-                    continue;
-                }
-
-                // Emulate PHP 8.0 T_NAME_* tokens, by combining sequences of T_NS_SEPARATOR and
-                // T_STRING into a single token.
-                if (($id === \T_NS_SEPARATOR || isset(self::$identifierTokens[$id]))) {
-                    $newText = $text;
-                    $lastWasSeparator = $id === \T_NS_SEPARATOR;
-                    for ($j = $i + 1; $j < $numTokens; $j++) {
-                        if ($lastWasSeparator) {
-                            if (!isset(self::$identifierTokens[$origTokens[$j][0]])) {
-                                break;
-                            }
-                            $lastWasSeparator = false;
-                        } else {
-                            if ($origTokens[$j][0] !== \T_NS_SEPARATOR) {
-                                break;
-                            }
-                            $lastWasSeparator = true;
-                        }
-                        $newText .= $origTokens[$j][1];
-                    }
-                    if ($lastWasSeparator) {
-                        // Trailing separator is not part of the name.
-                        $j--;
-                        $newText = \substr($newText, 0, -1);
-                    }
-                    if ($j > $i + 1) {
-                        if ($id === \T_NS_SEPARATOR) {
-                            $id = \T_NAME_FULLY_QUALIFIED;
-                        } elseif ($id === \T_NAMESPACE) {
-                            $id = \T_NAME_RELATIVE;
-                        } else {
-                            $id = \T_NAME_QUALIFIED;
-                        }
-                        $tokens[] = new static($id, $newText, $line, $pos);
-                        $pos += \strlen($newText);
-                        $i = $j - 1;
-                        continue;
-                    }
-                }
-
-                $tokens[] = new static($id, $text, $line, $pos);
-                $line += \substr_count($text, "\n");
-                $pos += \strlen($text);
-            }
-        }
-        return $tokens;
-    }
-
-    /** Initialize private static state needed by tokenize(). */
-    private static function init(): void {
-        if (isset(self::$identifierTokens)) {
-            return;
-        }
-
-        // Based on semi_reserved production.
-        self::$identifierTokens = \array_fill_keys([
-            \T_STRING,
-            \T_STATIC, \T_ABSTRACT, \T_FINAL, \T_PRIVATE, \T_PROTECTED, \T_PUBLIC, \T_READONLY,
-            \T_INCLUDE, \T_INCLUDE_ONCE, \T_EVAL, \T_REQUIRE, \T_REQUIRE_ONCE, \T_LOGICAL_OR, \T_LOGICAL_XOR, \T_LOGICAL_AND,
-            \T_INSTANCEOF, \T_NEW, \T_CLONE, \T_EXIT, \T_IF, \T_ELSEIF, \T_ELSE, \T_ENDIF, \T_ECHO, \T_DO, \T_WHILE,
-            \T_ENDWHILE, \T_FOR, \T_ENDFOR, \T_FOREACH, \T_ENDFOREACH, \T_DECLARE, \T_ENDDECLARE, \T_AS, \T_TRY, \T_CATCH,
-            \T_FINALLY, \T_THROW, \T_USE, \T_INSTEADOF, \T_GLOBAL, \T_VAR, \T_UNSET, \T_ISSET, \T_EMPTY, \T_CONTINUE, \T_GOTO,
-            \T_FUNCTION, \T_CONST, \T_RETURN, \T_PRINT, \T_YIELD, \T_LIST, \T_SWITCH, \T_ENDSWITCH, \T_CASE, \T_DEFAULT,
-            \T_BREAK, \T_ARRAY, \T_CALLABLE, \T_EXTENDS, \T_IMPLEMENTS, \T_NAMESPACE, \T_TRAIT, \T_INTERFACE, \T_CLASS,
-            \T_CLASS_C, \T_TRAIT_C, \T_FUNC_C, \T_METHOD_C, \T_LINE, \T_FILE, \T_DIR, \T_NS_C, \T_HALT_COMPILER, \T_FN,
-            \T_MATCH,
-        ], true);
-    }
-}

lib/PhpParser/Internal/TokenStream.php

@@ -2,36 +2,37 @@
 
 namespace PhpParser\Internal;
 
-use PhpParser\Token;
-
 /**
  * Provides operations on token streams, for use by pretty printer.
  *
  * @internal
  */
-class TokenStream {
+class TokenStream
-    /** @var Token[] Tokens (in PhpToken::tokenize() format) */
+{
-    private array $tokens;
+    /** @var array Tokens (in token_get_all format) */
+    private $tokens;
     /** @var int[] Map from position to indentation */
-    private array $indentMap;
+    private $indentMap;
 
     /**
      * Create token stream instance.
      *
-     * @param Token[] $tokens Tokens in PhpToken::tokenize() format
+     * @param array $tokens Tokens in token_get_all() format
      */
-    public function __construct(array $tokens, int $tabWidth) {
+    public function __construct(array $tokens) {
         $this->tokens = $tokens;
-        $this->indentMap = $this->calcIndentMap($tabWidth);
+        $this->indentMap = $this->calcIndentMap();
     }
 
     /**
      * Whether the given position is immediately surrounded by parenthesis.
      *
      * @param int $startPos Start position
-     * @param int $endPos End position
+     * @param int $endPos   End position
+     *
+     * @return bool
      */
-    public function haveParens(int $startPos, int $endPos): bool {
+    public function haveParens(int $startPos, int $endPos) : bool {
         return $this->haveTokenImmediatelyBefore($startPos, '(')
             && $this->haveTokenImmediatelyAfter($endPos, ')');
     }
@@ -40,9 +41,11 @@ class TokenStream {
      * Whether the given position is immediately surrounded by braces.
      *
      * @param int $startPos Start position
-     * @param int $endPos End position
+     * @param int $endPos   End position
+     *
+     * @return bool
      */
-    public function haveBraces(int $startPos, int $endPos): bool {
+    public function haveBraces(int $startPos, int $endPos) : bool {
         return ($this->haveTokenImmediatelyBefore($startPos, '{')
                 || $this->haveTokenImmediatelyBefore($startPos, T_CURLY_OPEN))
             && $this->haveTokenImmediatelyAfter($endPos, '}');
@@ -53,20 +56,21 @@ class TokenStream {
      *
      * During this check whitespace and comments are skipped.
      *
-     * @param int $pos Position before which the token should occur
+     * @param int        $pos               Position before which the token should occur
      * @param int|string $expectedTokenType Token to check for
      *
      * @return bool Whether the expected token was found
      */
-    public function haveTokenImmediatelyBefore(int $pos, $expectedTokenType): bool {
+    public function haveTokenImmediatelyBefore(int $pos, $expectedTokenType) : bool {
         $tokens = $this->tokens;
         $pos--;
         for (; $pos >= 0; $pos--) {
-            $token = $tokens[$pos];
+            $tokenType = $tokens[$pos][0];
-            if ($token->is($expectedTokenType)) {
+            if ($tokenType === $expectedTokenType) {
                 return true;
             }
-            if (!$token->isIgnorable()) {
+            if ($tokenType !== \T_WHITESPACE
+                && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
                 break;
             }
         }
@@ -78,28 +82,28 @@ class TokenStream {
      *
      * During this check whitespace and comments are skipped.
      *
-     * @param int $pos Position after which the token should occur
+     * @param int        $pos               Position after which the token should occur
      * @param int|string $expectedTokenType Token to check for
      *
      * @return bool Whether the expected token was found
      */
-    public function haveTokenImmediatelyAfter(int $pos, $expectedTokenType): bool {
+    public function haveTokenImmediatelyAfter(int $pos, $expectedTokenType) : bool {
         $tokens = $this->tokens;
         $pos++;
-        for ($c = \count($tokens); $pos < $c; $pos++) {
+        for (; $pos < \count($tokens); $pos++) {
-            $token = $tokens[$pos];
+            $tokenType = $tokens[$pos][0];
-            if ($token->is($expectedTokenType)) {
+            if ($tokenType === $expectedTokenType) {
                 return true;
             }
-            if (!$token->isIgnorable()) {
+            if ($tokenType !== \T_WHITESPACE
+                && $tokenType !== \T_COMMENT && $tokenType !== \T_DOC_COMMENT) {
                 break;
             }
         }
         return false;
     }
 
-    /** @param int|string|(int|string)[] $skipTokenType */
+    public function skipLeft(int $pos, $skipTokenType) {
-    public function skipLeft(int $pos, $skipTokenType): int {
         $tokens = $this->tokens;
 
         $pos = $this->skipLeftWhitespace($pos);
@@ -107,7 +111,7 @@ class TokenStream {
             return $pos;
         }
 
-        if (!$tokens[$pos]->is($skipTokenType)) {
+        if ($tokens[$pos][0] !== $skipTokenType) {
             // Shouldn't happen. The skip token MUST be there
             throw new \Exception('Encountered unexpected token');
         }
@@ -116,8 +120,7 @@ class TokenStream {
         return $this->skipLeftWhitespace($pos);
     }
 
-    /** @param int|string|(int|string)[] $skipTokenType */
+    public function skipRight(int $pos, $skipTokenType) {
-    public function skipRight(int $pos, $skipTokenType): int {
         $tokens = $this->tokens;
 
         $pos = $this->skipRightWhitespace($pos);
@@ -125,7 +128,7 @@ class TokenStream {
             return $pos;
         }
 
-        if (!$tokens[$pos]->is($skipTokenType)) {
+        if ($tokens[$pos][0] !== $skipTokenType) {
             // Shouldn't happen. The skip token MUST be there
             throw new \Exception('Encountered unexpected token');
         }
@@ -140,10 +143,11 @@ class TokenStream {
      * @param int $pos Token position
      * @return int Non-whitespace token position
      */
-    public function skipLeftWhitespace(int $pos): int {
+    public function skipLeftWhitespace(int $pos) {
         $tokens = $this->tokens;
         for (; $pos >= 0; $pos--) {
-            if (!$tokens[$pos]->isIgnorable()) {
+            $type = $tokens[$pos][0];
+            if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
         }
@@ -156,21 +160,22 @@ class TokenStream {
      * @param int $pos Token position
      * @return int Non-whitespace token position
      */
-    public function skipRightWhitespace(int $pos): int {
+    public function skipRightWhitespace(int $pos) {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
-            if (!$tokens[$pos]->isIgnorable()) {
+            $type = $tokens[$pos][0];
+            if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
         }
         return $pos;
     }
 
-    /** @param int|string|(int|string)[] $findTokenType */
+    public function findRight(int $pos, $findTokenType) {
-    public function findRight(int $pos, $findTokenType): int {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
-            if ($tokens[$pos]->is($findTokenType)) {
+            $type = $tokens[$pos][0];
+            if ($type === $findTokenType) {
                 return $pos;
             }
         }
@@ -185,16 +190,22 @@ class TokenStream {
      * @param int|string $tokenType Token type to look for
      * @return bool Whether the token occurs in the given range
      */
-    public function haveTokenInRange(int $startPos, int $endPos, $tokenType): bool {
+    public function haveTokenInRange(int $startPos, int $endPos, $tokenType) {
         $tokens = $this->tokens;
         for ($pos = $startPos; $pos < $endPos; $pos++) {
-            if ($tokens[$pos]->is($tokenType)) {
+            if ($tokens[$pos][0] === $tokenType) {
                 return true;
             }
         }
         return false;
     }
 
+    public function haveBracesInRange(int $startPos, int $endPos) {
+        return $this->haveTokenInRange($startPos, $endPos, '{')
+            || $this->haveTokenInRange($startPos, $endPos, T_CURLY_OPEN)
+            || $this->haveTokenInRange($startPos, $endPos, '}');
+    }
+
     public function haveTagInRange(int $startPos, int $endPos): bool {
         return $this->haveTokenInRange($startPos, $endPos, \T_OPEN_TAG)
             || $this->haveTokenInRange($startPos, $endPos, \T_CLOSE_TAG);
@@ -207,37 +218,41 @@ class TokenStream {
      *
      * @return int Indentation depth (in spaces)
      */
-    public function getIndentationBefore(int $pos): int {
+    public function getIndentationBefore(int $pos) : int {
         return $this->indentMap[$pos];
     }
 
     /**
      * Get the code corresponding to a token offset range, optionally adjusted for indentation.
      *
-     * @param int $from Token start position (inclusive)
+     * @param int $from   Token start position (inclusive)
-     * @param int $to Token end position (exclusive)
+     * @param int $to     Token end position (exclusive)
      * @param int $indent By how much the code should be indented (can be negative as well)
      *
      * @return string Code corresponding to token range, adjusted for indentation
      */
-    public function getTokenCode(int $from, int $to, int $indent): string {
+    public function getTokenCode(int $from, int $to, int $indent) : string {
         $tokens = $this->tokens;
         $result = '';
         for ($pos = $from; $pos < $to; $pos++) {
             $token = $tokens[$pos];
-            $id = $token->id;
+            if (\is_array($token)) {
-            $text = $token->text;
+                $type = $token[0];
-            if ($id === \T_CONSTANT_ENCAPSED_STRING || $id === \T_ENCAPSED_AND_WHITESPACE) {
+                $content = $token[1];
-                $result .= $text;
+                if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
-            } else {
+                    $result .= $content;
-                // TODO Handle non-space indentation
-                if ($indent < 0) {
-                    $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $text);
-                } elseif ($indent > 0) {
-                    $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $text);
                 } else {
-                    $result .= $text;
+                    // TODO Handle non-space indentation
+                    if ($indent < 0) {
+                        $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
+                    } elseif ($indent > 0) {
+                        $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
+                    } else {
+                        $result .= $content;
+                    }
                 }
+            } else {
+                $result .= $token;
             }
         }
         return $result;
@@ -248,21 +263,17 @@ class TokenStream {
      *
      * @return int[] Token position to indentation map
      */
-    private function calcIndentMap(int $tabWidth): array {
+    private function calcIndentMap() {
         $indentMap = [];
         $indent = 0;
-        foreach ($this->tokens as $i => $token) {
+        foreach ($this->tokens as $token) {
             $indentMap[] = $indent;
 
-            if ($token->id === \T_WHITESPACE) {
+            if ($token[0] === \T_WHITESPACE) {
-                $content = $token->text;
+                $content = $token[1];
                 $newlinePos = \strrpos($content, "\n");
                 if (false !== $newlinePos) {
-                    $indent = $this->getIndent(\substr($content, $newlinePos + 1), $tabWidth);
+                    $indent = \strlen($content) - $newlinePos - 1;
-                } elseif ($i === 1 && $this->tokens[0]->id === \T_OPEN_TAG &&
-                          $this->tokens[0]->text[\strlen($this->tokens[0]->text) - 1] === "\n") {
-                    // Special case: Newline at the end of opening tag followed by whitespace.
-                    $indent = $this->getIndent($content, $tabWidth);
                 }
             }
         }
@@ -272,11 +283,4 @@ class TokenStream {
 
         return $indentMap;
     }
-
-    private function getIndent(string $ws, int $tabWidth): int {
-        $spaces = \substr_count($ws, " ");
-        $tabs = \substr_count($ws, "\t");
-        assert(\strlen($ws) === $spaces + $tabs);
-        return $spaces + $tabs * $tabWidth;
-    }
 }

lib/PhpParser/JsonDecoder.php

@@ -2,11 +2,11 @@
 
 namespace PhpParser;
 
-class JsonDecoder {
+class JsonDecoder
-    /** @var \ReflectionClass<Node>[] Node type to reflection class map */
+{
-    private array $reflectionClassCache;
+    /** @var \ReflectionClass[] Node type to reflection class map */
+    private $reflectionClassCache;
 
-    /** @return mixed */
     public function decode(string $json) {
         $value = json_decode($json, true);
         if (json_last_error()) {
@@ -16,10 +16,6 @@ class JsonDecoder {
         return $this->decodeRecursive($value);
     }
 
-    /**
-     * @param mixed $value
-     * @return mixed
-     */
     private function decodeRecursive($value) {
         if (\is_array($value)) {
             if (isset($value['nodeType'])) {
@@ -33,7 +29,7 @@ class JsonDecoder {
         return $value;
     }
 
-    private function decodeArray(array $array): array {
+    private function decodeArray(array $array) : array {
         $decodedArray = [];
         foreach ($array as $key => $value) {
             $decodedArray[$key] = $this->decodeRecursive($value);
@@ -41,13 +37,14 @@ class JsonDecoder {
         return $decodedArray;
     }
 
-    private function decodeNode(array $value): Node {
+    private function decodeNode(array $value) : Node {
         $nodeType = $value['nodeType'];
         if (!\is_string($nodeType)) {
             throw new \RuntimeException('Node type must be a string');
         }
 
         $reflectionClass = $this->reflectionClassFromNodeType($nodeType);
+        /** @var Node $node */
         $node = $reflectionClass->newInstanceWithoutConstructor();
 
         if (isset($value['attributes'])) {
@@ -69,7 +66,7 @@ class JsonDecoder {
         return $node;
     }
 
-    private function decodeComment(array $value): Comment {
+    private function decodeComment(array $value) : Comment {
         $className = $value['nodeType'] === 'Comment' ? Comment::class : Comment\Doc::class;
         if (!isset($value['text'])) {
             throw new \RuntimeException('Comment must have text');
@@ -82,8 +79,7 @@ class JsonDecoder {
         );
     }
 
-    /** @return \ReflectionClass<Node> */
+    private function reflectionClassFromNodeType(string $nodeType) : \ReflectionClass {
-    private function reflectionClassFromNodeType(string $nodeType): \ReflectionClass {
         if (!isset($this->reflectionClassCache[$nodeType])) {
             $className = $this->classNameFromNodeType($nodeType);
             $this->reflectionClassCache[$nodeType] = new \ReflectionClass($className);
@@ -91,8 +87,7 @@ class JsonDecoder {
         return $this->reflectionClassCache[$nodeType];
     }
 
-    /** @return class-string<Node> */
+    private function classNameFromNodeType(string $nodeType) : string {
-    private function classNameFromNodeType(string $nodeType): string {
         $className = 'PhpParser\\Node\\' . strtr($nodeType, '_', '\\');
         if (class_exists($className)) {
             return $className;

lib/PhpParser/Lexer.php

@@ -2,115 +2,559 @@
 
 namespace PhpParser;
 
-require __DIR__ . '/compatibility_tokens.php';
+use PhpParser\Parser\Tokens;
+
+class Lexer
+{
+    protected $code;
+    protected $tokens;
+    protected $pos;
+    protected $line;
+    protected $filePos;
+    protected $prevCloseTagHasNewline;
+
+    protected $tokenMap;
+    protected $dropTokens;
+    protected $identifierTokens;
+
+    private $attributeStartLineUsed;
+    private $attributeEndLineUsed;
+    private $attributeStartTokenPosUsed;
+    private $attributeEndTokenPosUsed;
+    private $attributeStartFilePosUsed;
+    private $attributeEndFilePosUsed;
+    private $attributeCommentsUsed;
 
-class Lexer {
     /**
-     * Tokenize the provided source code.
+     * Creates a Lexer.
      *
-     * The token array is in the same format as provided by the PhpToken::tokenize() method in
+     * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
-     * PHP 8.0. The tokens are instances of PhpParser\Token, to abstract over a polyfill
+     *                       which is an array of attributes to add to the AST nodes. Possible
-     * implementation in earlier PHP version.
+     *                       attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
-     *
+     *                       'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
-     * The token array is terminated by a sentinel token with token ID 0.
+     *                       first three. For more info see getNextToken() docs.
-     * The token array does not discard any tokens (i.e. whitespace and comments are included).
-     * The token position attributes are against this token array.
-     *
-     * @param string $code The source code to tokenize.
-     * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
-     *                                        ErrorHandler\Throwing.
-     * @return Token[] Tokens
      */
-    public function tokenize(string $code, ?ErrorHandler $errorHandler = null): array {
+    public function __construct(array $options = []) {
+        // Create Map from internal tokens to PhpParser tokens.
+        $this->defineCompatibilityTokens();
+        $this->tokenMap = $this->createTokenMap();
+        $this->identifierTokens = $this->createIdentifierTokenMap();
+
+        // map of tokens to drop while lexing (the map is only used for isset lookup,
+        // that's why the value is simply set to 1; the value is never actually used.)
+        $this->dropTokens = array_fill_keys(
+            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
+        );
+
+        $defaultAttributes = ['comments', 'startLine', 'endLine'];
+        $usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
+
+        // Create individual boolean properties to make these checks faster.
+        $this->attributeStartLineUsed = isset($usedAttributes['startLine']);
+        $this->attributeEndLineUsed = isset($usedAttributes['endLine']);
+        $this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
+        $this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
+        $this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
+        $this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
+        $this->attributeCommentsUsed = isset($usedAttributes['comments']);
+    }
+
+    /**
+     * Initializes the lexer for lexing the provided source code.
+     *
+     * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
+     * the getErrors() method.
+     *
+     * @param string $code The source code to lex
+     * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
+     *                                        ErrorHandler\Throwing
+     */
+    public function startLexing(string $code, ?ErrorHandler $errorHandler = null) {
         if (null === $errorHandler) {
             $errorHandler = new ErrorHandler\Throwing();
         }
 
+        $this->code = $code; // keep the code around for __halt_compiler() handling
+        $this->pos  = -1;
+        $this->line =  1;
+        $this->filePos = 0;
+
+        // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
+        // This ensures proper composability, because having a newline is the "safe" assumption.
+        $this->prevCloseTagHasNewline = true;
+
         $scream = ini_set('xdebug.scream', '0');
 
-        $tokens = @Token::tokenize($code);
+        $this->tokens = @token_get_all($code);
-        $this->postprocessTokens($tokens, $errorHandler);
+        $this->postprocessTokens($errorHandler);
 
         if (false !== $scream) {
             ini_set('xdebug.scream', $scream);
         }
+    }
 
+    private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
+        $tokens = [];
+        for ($i = $start; $i < $end; $i++) {
+            $chr = $this->code[$i];
+            if ($chr === "\0") {
+                // PHP cuts error message after null byte, so need special case
+                $errorMsg = 'Unexpected null byte';
+            } else {
+                $errorMsg = sprintf(
+                    'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
+                );
+            }
+
+            $tokens[] = [\T_BAD_CHARACTER, $chr, $line];
+            $errorHandler->handleError(new Error($errorMsg, [
+                'startLine' => $line,
+                'endLine' => $line,
+                'startFilePos' => $i,
+                'endFilePos' => $i,
+            ]));
+        }
         return $tokens;
     }
 
-    private function handleInvalidCharacter(Token $token, ErrorHandler $errorHandler): void {
-        $chr = $token->text;
-        if ($chr === "\0") {
-            // PHP cuts error message after null byte, so need special case
-            $errorMsg = 'Unexpected null byte';
-        } else {
-            $errorMsg = sprintf(
-                'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
-            );
-        }
-
-        $errorHandler->handleError(new Error($errorMsg, [
-            'startLine' => $token->line,
-            'endLine' => $token->line,
-            'startFilePos' => $token->pos,
-            'endFilePos' => $token->pos,
-        ]));
-    }
-
-    private function isUnterminatedComment(Token $token): bool {
-        return $token->is([\T_COMMENT, \T_DOC_COMMENT])
-            && substr($token->text, 0, 2) === '/*'
-            && substr($token->text, -2) !== '*/';
-    }
-
     /**
-     * @param list<Token> $tokens
+     * Check whether comment token is unterminated.
+     *
+     * @return bool
      */
-    protected function postprocessTokens(array &$tokens, ErrorHandler $errorHandler): void {
+    private function isUnterminatedComment($token) : bool {
-        // This function reports errors (bad characters and unterminated comments) in the token
+        return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
-        // array, and performs certain canonicalizations:
+            && substr($token[1], 0, 2) === '/*'
+            && substr($token[1], -2) !== '*/';
+    }
+
+    protected function postprocessTokens(ErrorHandler $errorHandler) {
+        // PHP's error handling for token_get_all() is rather bad, so if we want detailed
+        // error information we need to compute it ourselves. Invalid character errors are
+        // detected by finding "gaps" in the token array. Unterminated comments are detected
+        // by checking if a trailing comment has a "*/" at the end.
+        //
+        // Additionally, we perform a number of canonicalizations here:
+        //  * Use the PHP 8.0 comment format, which does not include trailing whitespace anymore.
+        //  * Use PHP 8.0 T_NAME_* tokens.
         //  * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
         //    T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
-        //  * Add a sentinel token with ID 0.
 
-        $numTokens = \count($tokens);
+        $filePos = 0;
-        if ($numTokens === 0) {
+        $line = 1;
-            // Empty input edge case: Just add the sentinel token.
+        $numTokens = \count($this->tokens);
-            $tokens[] = new Token(0, "\0", 1, 0);
+        for ($i = 0; $i < $numTokens; $i++) {
+            $token = $this->tokens[$i];
+
+            // Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
+            // In this case we only need to emit an error.
+            if ($token[0] === \T_BAD_CHARACTER) {
+                $this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
+            }
+
+            if ($token[0] === \T_COMMENT && substr($token[1], 0, 2) !== '/*'
+                    && preg_match('/(\r\n|\n|\r)$/D', $token[1], $matches)) {
+                $trailingNewline = $matches[0];
+                $token[1] = substr($token[1], 0, -strlen($trailingNewline));
+                $this->tokens[$i] = $token;
+                if (isset($this->tokens[$i + 1]) && $this->tokens[$i + 1][0] === \T_WHITESPACE) {
+                    // Move trailing newline into following T_WHITESPACE token, if it already exists.
+                    $this->tokens[$i + 1][1] = $trailingNewline . $this->tokens[$i + 1][1];
+                    $this->tokens[$i + 1][2]--;
+                } else {
+                    // Otherwise, we need to create a new T_WHITESPACE token.
+                    array_splice($this->tokens, $i + 1, 0, [
+                        [\T_WHITESPACE, $trailingNewline, $line],
+                    ]);
+                    $numTokens++;
+                }
+            }
+
+            // Emulate PHP 8 T_NAME_* tokens, by combining sequences of T_NS_SEPARATOR and T_STRING
+            // into a single token.
+            if (\is_array($token)
+                    && ($token[0] === \T_NS_SEPARATOR || isset($this->identifierTokens[$token[0]]))) {
+                $lastWasSeparator = $token[0] === \T_NS_SEPARATOR;
+                $text = $token[1];
+                for ($j = $i + 1; isset($this->tokens[$j]); $j++) {
+                    if ($lastWasSeparator) {
+                        if (!isset($this->identifierTokens[$this->tokens[$j][0]])) {
+                            break;
+                        }
+                        $lastWasSeparator = false;
+                    } else {
+                        if ($this->tokens[$j][0] !== \T_NS_SEPARATOR) {
+                            break;
+                        }
+                        $lastWasSeparator = true;
+                    }
+                    $text .= $this->tokens[$j][1];
+                }
+                if ($lastWasSeparator) {
+                    // Trailing separator is not part of the name.
+                    $j--;
+                    $text = substr($text, 0, -1);
+                }
+                if ($j > $i + 1) {
+                    if ($token[0] === \T_NS_SEPARATOR) {
+                        $type = \T_NAME_FULLY_QUALIFIED;
+                    } else if ($token[0] === \T_NAMESPACE) {
+                        $type = \T_NAME_RELATIVE;
+                    } else {
+                        $type = \T_NAME_QUALIFIED;
+                    }
+                    $token = [$type, $text, $line];
+                    array_splice($this->tokens, $i, $j - $i, [$token]);
+                    $numTokens -= $j - $i - 1;
+                }
+            }
+
+            if ($token === '&') {
+                $next = $i + 1;
+                while (isset($this->tokens[$next]) && $this->tokens[$next][0] === \T_WHITESPACE) {
+                    $next++;
+                }
+                $followedByVarOrVarArg = isset($this->tokens[$next]) &&
+                    ($this->tokens[$next][0] === \T_VARIABLE || $this->tokens[$next][0] === \T_ELLIPSIS);
+                $this->tokens[$i] = $token = [
+                    $followedByVarOrVarArg
+                        ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
+                        : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG,
+                    '&',
+                    $line,
+                ];
+            }
+
+            $tokenValue = \is_string($token) ? $token : $token[1];
+            $tokenLen = \strlen($tokenValue);
+
+            if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
+                // Something is missing, must be an invalid character
+                $nextFilePos = strpos($this->code, $tokenValue, $filePos);
+                $badCharTokens = $this->handleInvalidCharacterRange(
+                    $filePos, $nextFilePos, $line, $errorHandler);
+                $filePos = (int) $nextFilePos;
+
+                array_splice($this->tokens, $i, 0, $badCharTokens);
+                $numTokens += \count($badCharTokens);
+                $i += \count($badCharTokens);
+            }
+
+            $filePos += $tokenLen;
+            $line += substr_count($tokenValue, "\n");
+        }
+
+        if ($filePos !== \strlen($this->code)) {
+            if (substr($this->code, $filePos, 2) === '/*') {
+                // Unlike PHP, HHVM will drop unterminated comments entirely
+                $comment = substr($this->code, $filePos);
+                $errorHandler->handleError(new Error('Unterminated comment', [
+                    'startLine' => $line,
+                    'endLine' => $line + substr_count($comment, "\n"),
+                    'startFilePos' => $filePos,
+                    'endFilePos' => $filePos + \strlen($comment),
+                ]));
+
+                // Emulate the PHP behavior
+                $isDocComment = isset($comment[3]) && $comment[3] === '*';
+                $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
+            } else {
+                // Invalid characters at the end of the input
+                $badCharTokens = $this->handleInvalidCharacterRange(
+                    $filePos, \strlen($this->code), $line, $errorHandler);
+                $this->tokens = array_merge($this->tokens, $badCharTokens);
+            }
             return;
         }
 
-        for ($i = 0; $i < $numTokens; $i++) {
+        if (count($this->tokens) > 0) {
-            $token = $tokens[$i];
+            // Check for unterminated comment
-            if ($token->id === \T_BAD_CHARACTER) {
+            $lastToken = $this->tokens[count($this->tokens) - 1];
-                $this->handleInvalidCharacter($token, $errorHandler);
+            if ($this->isUnterminatedComment($lastToken)) {
+                $errorHandler->handleError(new Error('Unterminated comment', [
+                    'startLine' => $line - substr_count($lastToken[1], "\n"),
+                    'endLine' => $line,
+                    'startFilePos' => $filePos - \strlen($lastToken[1]),
+                    'endFilePos' => $filePos,
+                ]));
+            }
+        }
+    }
+
+    /**
+     * Fetches the next token.
+     *
+     * The available attributes are determined by the 'usedAttributes' option, which can
+     * be specified in the constructor. The following attributes are supported:
+     *
+     *  * 'comments'      => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
+     *                       representing all comments that occurred between the previous
+     *                       non-discarded token and the current one.
+     *  * 'startLine'     => Line in which the node starts.
+     *  * 'endLine'       => Line in which the node ends.
+     *  * 'startTokenPos' => Offset into the token array of the first token in the node.
+     *  * 'endTokenPos'   => Offset into the token array of the last token in the node.
+     *  * 'startFilePos'  => Offset into the code string of the first character that is part of the node.
+     *  * 'endFilePos'    => Offset into the code string of the last character that is part of the node.
+     *
+     * @param mixed $value           Variable to store token content in
+     * @param mixed $startAttributes Variable to store start attributes in
+     * @param mixed $endAttributes   Variable to store end attributes in
+     *
+     * @return int Token id
+     */
+    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
+        $startAttributes = [];
+        $endAttributes   = [];
+
+        while (1) {
+            if (isset($this->tokens[++$this->pos])) {
+                $token = $this->tokens[$this->pos];
+            } else {
+                // EOF token with ID 0
+                $token = "\0";
             }
 
-            if ($token->id === \ord('&')) {
+            if ($this->attributeStartLineUsed) {
-                $next = $i + 1;
+                $startAttributes['startLine'] = $this->line;
-                while (isset($tokens[$next]) && $tokens[$next]->id === \T_WHITESPACE) {
+            }
-                    $next++;
+            if ($this->attributeStartTokenPosUsed) {
+                $startAttributes['startTokenPos'] = $this->pos;
+            }
+            if ($this->attributeStartFilePosUsed) {
+                $startAttributes['startFilePos'] = $this->filePos;
+            }
+
+            if (\is_string($token)) {
+                $value = $token;
+                if (isset($token[1])) {
+                    // bug in token_get_all
+                    $this->filePos += 2;
+                    $id = ord('"');
+                } else {
+                    $this->filePos += 1;
+                    $id = ord($token);
                 }
-                $followedByVarOrVarArg = isset($tokens[$next]) &&
+            } elseif (!isset($this->dropTokens[$token[0]])) {
-                    $tokens[$next]->is([\T_VARIABLE, \T_ELLIPSIS]);
+                $value = $token[1];
-                $token->id = $followedByVarOrVarArg
+                $id = $this->tokenMap[$token[0]];
-                    ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
+                if (\T_CLOSE_TAG === $token[0]) {
-                    : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
+                    $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n")
+                        || false !== strpos($token[1], "\r");
+                } elseif (\T_INLINE_HTML === $token[0]) {
+                    $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
+                }
+
+                $this->line += substr_count($value, "\n");
+                $this->filePos += \strlen($value);
+            } else {
+                $origLine = $this->line;
+                $origFilePos = $this->filePos;
+                $this->line += substr_count($token[1], "\n");
+                $this->filePos += \strlen($token[1]);
+
+                if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
+                    if ($this->attributeCommentsUsed) {
+                        $comment = \T_DOC_COMMENT === $token[0]
+                            ? new Comment\Doc($token[1],
+                                $origLine, $origFilePos, $this->pos,
+                                $this->line, $this->filePos - 1, $this->pos)
+                            : new Comment($token[1],
+                                $origLine, $origFilePos, $this->pos,
+                                $this->line, $this->filePos - 1, $this->pos);
+                        $startAttributes['comments'][] = $comment;
+                    }
+                }
+                continue;
+            }
+
+            if ($this->attributeEndLineUsed) {
+                $endAttributes['endLine'] = $this->line;
+            }
+            if ($this->attributeEndTokenPosUsed) {
+                $endAttributes['endTokenPos'] = $this->pos;
+            }
+            if ($this->attributeEndFilePosUsed) {
+                $endAttributes['endFilePos'] = $this->filePos - 1;
+            }
+
+            return $id;
+        }
+
+        throw new \RuntimeException('Reached end of lexer loop');
+    }
+
+    /**
+     * Returns the token array for current code.
+     *
+     * The token array is in the same format as provided by the
+     * token_get_all() function and does not discard tokens (i.e.
+     * whitespace and comments are included). The token position
+     * attributes are against this token array.
+     *
+     * @return array Array of tokens in token_get_all() format
+     */
+    public function getTokens() : array {
+        return $this->tokens;
+    }
+
+    /**
+     * Handles __halt_compiler() by returning the text after it.
+     *
+     * @return string Remaining text
+     */
+    public function handleHaltCompiler() : string {
+        // text after T_HALT_COMPILER, still including ();
+        $textAfter = substr($this->code, $this->filePos);
+
+        // ensure that it is followed by ();
+        // this simplifies the situation, by not allowing any comments
+        // in between of the tokens.
+        if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
+            throw new Error('__HALT_COMPILER must be followed by "();"');
+        }
+
+        // prevent the lexer from returning any further tokens
+        $this->pos = count($this->tokens);
+
+        // return with (); removed
+        return substr($textAfter, strlen($matches[0]));
+    }
+
+    private function defineCompatibilityTokens() {
+        static $compatTokensDefined = false;
+        if ($compatTokensDefined) {
+            return;
+        }
+
+        $compatTokens = [
+            // PHP 7.4
+            'T_BAD_CHARACTER',
+            'T_FN',
+            'T_COALESCE_EQUAL',
+            // PHP 8.0
+            'T_NAME_QUALIFIED',
+            'T_NAME_FULLY_QUALIFIED',
+            'T_NAME_RELATIVE',
+            'T_MATCH',
+            'T_NULLSAFE_OBJECT_OPERATOR',
+            'T_ATTRIBUTE',
+            // PHP 8.1
+            'T_ENUM',
+            'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
+            'T_READONLY',
+        ];
+
+        // PHP-Parser might be used together with another library that also emulates some or all
+        // of these tokens. Perform a sanity-check that all already defined tokens have been
+        // assigned a unique ID.
+        $usedTokenIds = [];
+        foreach ($compatTokens as $token) {
+            if (\defined($token)) {
+                $tokenId = \constant($token);
+                $clashingToken = $usedTokenIds[$tokenId] ?? null;
+                if ($clashingToken !== null) {
+                    throw new \Error(sprintf(
+                        'Token %s has same ID as token %s, ' .
+                        'you may be using a library with broken token emulation',
+                        $token, $clashingToken
+                    ));
+                }
+                $usedTokenIds[$tokenId] = $token;
             }
         }
 
-        // Check for unterminated comment
+        // Now define any tokens that have not yet been emulated. Try to assign IDs from -1
-        $lastToken = $tokens[$numTokens - 1];
+        // downwards, but skip any IDs that may already be in use.
-        if ($this->isUnterminatedComment($lastToken)) {
+        $newTokenId = -1;
-            $errorHandler->handleError(new Error('Unterminated comment', [
+        foreach ($compatTokens as $token) {
-                'startLine' => $lastToken->line,
+            if (!\defined($token)) {
-                'endLine' => $lastToken->getEndLine(),
+                while (isset($usedTokenIds[$newTokenId])) {
-                'startFilePos' => $lastToken->pos,
+                    $newTokenId--;
-                'endFilePos' => $lastToken->getEndPos(),
+                }
-            ]));
+                \define($token, $newTokenId);
+                $newTokenId--;
+            }
         }
 
-        // Add sentinel token.
+        $compatTokensDefined = true;
-        $tokens[] = new Token(0, "\0", $lastToken->getEndLine(), $lastToken->getEndPos());
+    }
+
+    /**
+     * Creates the token map.
+     *
+     * The token map maps the PHP internal token identifiers
+     * to the identifiers used by the Parser. Additionally it
+     * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
+     *
+     * @return array The token map
+     */
+    protected function createTokenMap() : array {
+        $tokenMap = [];
+
+        // 256 is the minimum possible token number, as everything below
+        // it is an ASCII value
+        for ($i = 256; $i < 1000; ++$i) {
+            if (\T_DOUBLE_COLON === $i) {
+                // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
+                $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
+            } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
+                // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
+                $tokenMap[$i] = Tokens::T_ECHO;
+            } elseif(\T_CLOSE_TAG === $i) {
+                // T_CLOSE_TAG is equivalent to ';'
+                $tokenMap[$i] = ord(';');
+            } elseif ('UNKNOWN' !== $name = token_name($i)) {
+                if ('T_HASHBANG' === $name) {
+                    // HHVM uses a special token for #! hashbang lines
+                    $tokenMap[$i] = Tokens::T_INLINE_HTML;
+                } elseif (defined($name = Tokens::class . '::' . $name)) {
+                    // Other tokens can be mapped directly
+                    $tokenMap[$i] = constant($name);
+                }
+            }
+        }
+
+        // HHVM uses a special token for numbers that overflow to double
+        if (defined('T_ONUMBER')) {
+            $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
+        }
+        // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
+        if (defined('T_COMPILER_HALT_OFFSET')) {
+            $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
+        }
+
+        // Assign tokens for which we define compatibility constants, as token_name() does not know them.
+        $tokenMap[\T_FN] = Tokens::T_FN;
+        $tokenMap[\T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
+        $tokenMap[\T_NAME_QUALIFIED] = Tokens::T_NAME_QUALIFIED;
+        $tokenMap[\T_NAME_FULLY_QUALIFIED] = Tokens::T_NAME_FULLY_QUALIFIED;
+        $tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
+        $tokenMap[\T_MATCH] = Tokens::T_MATCH;
+        $tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR;
+        $tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE;
+        $tokenMap[\T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
+        $tokenMap[\T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG;
+        $tokenMap[\T_ENUM] = Tokens::T_ENUM;
+        $tokenMap[\T_READONLY] = Tokens::T_READONLY;
+
+        return $tokenMap;
+    }
+
+    private function createIdentifierTokenMap(): array {
+        // Based on semi_reserved production.
+        return array_fill_keys([
+            \T_STRING,
+            \T_STATIC, \T_ABSTRACT, \T_FINAL, \T_PRIVATE, \T_PROTECTED, \T_PUBLIC, \T_READONLY,
+            \T_INCLUDE, \T_INCLUDE_ONCE, \T_EVAL, \T_REQUIRE, \T_REQUIRE_ONCE, \T_LOGICAL_OR, \T_LOGICAL_XOR, \T_LOGICAL_AND,
+            \T_INSTANCEOF, \T_NEW, \T_CLONE, \T_EXIT, \T_IF, \T_ELSEIF, \T_ELSE, \T_ENDIF, \T_ECHO, \T_DO, \T_WHILE,
+            \T_ENDWHILE, \T_FOR, \T_ENDFOR, \T_FOREACH, \T_ENDFOREACH, \T_DECLARE, \T_ENDDECLARE, \T_AS, \T_TRY, \T_CATCH,
+            \T_FINALLY, \T_THROW, \T_USE, \T_INSTEADOF, \T_GLOBAL, \T_VAR, \T_UNSET, \T_ISSET, \T_EMPTY, \T_CONTINUE, \T_GOTO,
+            \T_FUNCTION, \T_CONST, \T_RETURN, \T_PRINT, \T_YIELD, \T_LIST, \T_SWITCH, \T_ENDSWITCH, \T_CASE, \T_DEFAULT,
+            \T_BREAK, \T_ARRAY, \T_CALLABLE, \T_EXTENDS, \T_IMPLEMENTS, \T_NAMESPACE, \T_TRAIT, \T_INTERFACE, \T_CLASS,
+            \T_CLASS_C, \T_TRAIT_C, \T_FUNC_C, \T_METHOD_C, \T_LINE, \T_FILE, \T_DIR, \T_NS_C, \T_HALT_COMPILER, \T_FN,
+            \T_MATCH,
+        ], true);
     }
 }

lib/PhpParser/Lexer/Emulative.php

@@ -5,48 +5,61 @@ namespace PhpParser\Lexer;
 use PhpParser\Error;
 use PhpParser\ErrorHandler;
 use PhpParser\Lexer;
-use PhpParser\Lexer\TokenEmulator\AsymmetricVisibilityTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\AttributeEmulator;
 use PhpParser\Lexer\TokenEmulator\EnumTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\ExplicitOctalEmulator;
+use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator;
+use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\MatchTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\NullsafeTokenEmulator;
-use PhpParser\Lexer\TokenEmulator\PropertyTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
 use PhpParser\Lexer\TokenEmulator\ReadonlyFunctionTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\ReadonlyTokenEmulator;
 use PhpParser\Lexer\TokenEmulator\ReverseEmulator;
 use PhpParser\Lexer\TokenEmulator\TokenEmulator;
-use PhpParser\PhpVersion;
-use PhpParser\Token;
 
-class Emulative extends Lexer {
+class Emulative extends Lexer
-    /** @var array{int, string, string}[] Patches used to reverse changes introduced in the code */
+{
-    private array $patches = [];
+    const PHP_7_3 = '7.3dev';
+    const PHP_7_4 = '7.4dev';
+    const PHP_8_0 = '8.0dev';
+    const PHP_8_1 = '8.1dev';
+    const PHP_8_2 = '8.2dev';
 
-    /** @var list<TokenEmulator> */
+    /** @var mixed[] Patches used to reverse changes introduced in the code */
-    private array $emulators = [];
+    private $patches = [];
 
-    private PhpVersion $targetPhpVersion;
+    /** @var TokenEmulator[] */
+    private $emulators = [];
 
-    private PhpVersion $hostPhpVersion;
+    /** @var string */
+    private $targetPhpVersion;
 
     /**
-     * @param PhpVersion|null $phpVersion PHP version to emulate. Defaults to newest supported.
+     * @param mixed[] $options Lexer options. In addition to the usual options,
+     *                         accepts a 'phpVersion' string that specifies the
+     *                         version to emulate. Defaults to newest supported.
      */
-    public function __construct(?PhpVersion $phpVersion = null) {
+    public function __construct(array $options = [])
-        $this->targetPhpVersion = $phpVersion ?? PhpVersion::getNewestSupported();
+    {
-        $this->hostPhpVersion = PhpVersion::getHostVersion();
+        $this->targetPhpVersion = $options['phpVersion'] ?? Emulative::PHP_8_2;
+        unset($options['phpVersion']);
+
+        parent::__construct($options);
 
         $emulators = [
+            new FlexibleDocStringEmulator(),
+            new FnTokenEmulator(),
             new MatchTokenEmulator(),
+            new CoaleseEqualTokenEmulator(),
+            new NumericLiteralSeparatorEmulator(),
             new NullsafeTokenEmulator(),
             new AttributeEmulator(),
             new EnumTokenEmulator(),
             new ReadonlyTokenEmulator(),
             new ExplicitOctalEmulator(),
             new ReadonlyFunctionTokenEmulator(),
-            new PropertyTokenEmulator(),
-            new AsymmetricVisibilityTokenEmulator(),
         ];
 
         // Collect emulators that are relevant for the PHP version we're running
@@ -55,24 +68,21 @@ class Emulative extends Lexer {
             $emulatorPhpVersion = $emulator->getPhpVersion();
             if ($this->isForwardEmulationNeeded($emulatorPhpVersion)) {
                 $this->emulators[] = $emulator;
-            } elseif ($this->isReverseEmulationNeeded($emulatorPhpVersion)) {
+            } else if ($this->isReverseEmulationNeeded($emulatorPhpVersion)) {
                 $this->emulators[] = new ReverseEmulator($emulator);
             }
         }
     }
 
-    public function tokenize(string $code, ?ErrorHandler $errorHandler = null): array {
+    public function startLexing(string $code, ?ErrorHandler $errorHandler = null) {
-        $emulators = array_filter($this->emulators, function ($emulator) use ($code) {
+        $emulators = array_filter($this->emulators, function($emulator) use($code) {
             return $emulator->isEmulationNeeded($code);
         });
 
         if (empty($emulators)) {
             // Nothing to emulate, yay
-            return parent::tokenize($code, $errorHandler);
+            parent::startLexing($code, $errorHandler);
-        }
+            return;
-
-        if ($errorHandler === null) {
-            $errorHandler = new ErrorHandler\Throwing();
         }
 
         $this->patches = [];
@@ -81,9 +91,9 @@ class Emulative extends Lexer {
         }
 
         $collector = new ErrorHandler\Collecting();
-        $tokens = parent::tokenize($code, $collector);
+        parent::startLexing($code, $collector);
         $this->sortPatches();
-        $tokens = $this->fixupTokens($tokens);
+        $this->fixupTokens();
 
         $errors = $collector->getErrors();
         if (!empty($errors)) {
@@ -94,80 +104,90 @@ class Emulative extends Lexer {
         }
 
         foreach ($emulators as $emulator) {
-            $tokens = $emulator->emulate($code, $tokens);
+            $this->tokens = $emulator->emulate($code, $this->tokens);
         }
-
-        return $tokens;
     }
 
-    private function isForwardEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
+    private function isForwardEmulationNeeded(string $emulatorPhpVersion): bool {
-        return $this->hostPhpVersion->older($emulatorPhpVersion)
+        return version_compare(\PHP_VERSION, $emulatorPhpVersion, '<')
-            && $this->targetPhpVersion->newerOrEqual($emulatorPhpVersion);
+            && version_compare($this->targetPhpVersion, $emulatorPhpVersion, '>=');
     }
 
-    private function isReverseEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
+    private function isReverseEmulationNeeded(string $emulatorPhpVersion): bool {
-        return $this->hostPhpVersion->newerOrEqual($emulatorPhpVersion)
+        return version_compare(\PHP_VERSION, $emulatorPhpVersion, '>=')
-            && $this->targetPhpVersion->older($emulatorPhpVersion);
+            && version_compare($this->targetPhpVersion, $emulatorPhpVersion, '<');
     }
 
-    private function sortPatches(): void {
+    private function sortPatches()
+    {
         // Patches may be contributed by different emulators.
         // Make sure they are sorted by increasing patch position.
-        usort($this->patches, function ($p1, $p2) {
+        usort($this->patches, function($p1, $p2) {
             return $p1[0] <=> $p2[0];
         });
     }
 
-    /**
+    private function fixupTokens()
-     * @param list<Token> $tokens
+    {
-     * @return list<Token>
-     */
-    private function fixupTokens(array $tokens): array {
         if (\count($this->patches) === 0) {
-            return $tokens;
+            return;
         }
 
         // Load first patch
         $patchIdx = 0;
+
         list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
 
         // We use a manual loop over the tokens, because we modify the array on the fly
-        $posDelta = 0;
+        $pos = 0;
-        $lineDelta = 0;
+        for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
-        for ($i = 0, $c = \count($tokens); $i < $c; $i++) {
+            $token = $this->tokens[$i];
-            $token = $tokens[$i];
+            if (\is_string($token)) {
-            $pos = $token->pos;
+                if ($patchPos === $pos) {
-            $token->pos += $posDelta;
+                    // Only support replacement for string tokens.
-            $token->line += $lineDelta;
+                    assert($patchType === 'replace');
-            $localPosDelta = 0;
+                    $this->tokens[$i] = $patchText;
-            $len = \strlen($token->text);
+
+                    // Fetch the next patch
+                    $patchIdx++;
+                    if ($patchIdx >= \count($this->patches)) {
+                        // No more patches, we're done
+                        return;
+                    }
+                    list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+                }
+
+                $pos += \strlen($token);
+                continue;
+            }
+
+            $len = \strlen($token[1]);
+            $posDelta = 0;
             while ($patchPos >= $pos && $patchPos < $pos + $len) {
                 $patchTextLen = \strlen($patchText);
                 if ($patchType === 'remove') {
                     if ($patchPos === $pos && $patchTextLen === $len) {
                         // Remove token entirely
-                        array_splice($tokens, $i, 1, []);
+                        array_splice($this->tokens, $i, 1, []);
                         $i--;
                         $c--;
                     } else {
                         // Remove from token string
-                        $token->text = substr_replace(
+                        $this->tokens[$i][1] = substr_replace(
-                            $token->text, '', $patchPos - $pos + $localPosDelta, $patchTextLen
+                            $token[1], '', $patchPos - $pos + $posDelta, $patchTextLen
                         );
-                        $localPosDelta -= $patchTextLen;
+                        $posDelta -= $patchTextLen;
                     }
-                    $lineDelta -= \substr_count($patchText, "\n");
                 } elseif ($patchType === 'add') {
                     // Insert into the token string
-                    $token->text = substr_replace(
+                    $this->tokens[$i][1] = substr_replace(
-                        $token->text, $patchText, $patchPos - $pos + $localPosDelta, 0
+                        $token[1], $patchText, $patchPos - $pos + $posDelta, 0
                     );
-                    $localPosDelta += $patchTextLen;
+                    $posDelta += $patchTextLen;
-                    $lineDelta += \substr_count($patchText, "\n");
+                } else if ($patchType === 'replace') {
-                } elseif ($patchType === 'replace') {
                     // Replace inside the token string
-                    $token->text = substr_replace(
+                    $this->tokens[$i][1] = substr_replace(
-                        $token->text, $patchText, $patchPos - $pos + $localPosDelta, $patchTextLen
+                        $token[1], $patchText, $patchPos - $pos + $posDelta, $patchTextLen
                     );
                 } else {
                     assert(false);
@@ -176,17 +196,22 @@ class Emulative extends Lexer {
                 // Fetch the next patch
                 $patchIdx++;
                 if ($patchIdx >= \count($this->patches)) {
-                    // No more patches. However, we still need to adjust position.
+                    // No more patches, we're done
-                    $patchPos = \PHP_INT_MAX;
+                    return;
-                    break;
                 }
 
                 list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+
+                // Multiple patches may apply to the same token. Reload the current one to check
+                // If the new patch applies
+                $token = $this->tokens[$i];
             }
 
-            $posDelta += $localPosDelta;
+            $pos += $len;
         }
-        return $tokens;
+
+        // A patch did not apply
+        assert(false);
     }
 
     /**
@@ -194,7 +219,7 @@ class Emulative extends Lexer {
      *
      * @param Error[] $errors
      */
-    private function fixupErrors(array $errors): void {
+    private function fixupErrors(array $errors) {
         foreach ($errors as $error) {
             $attrs = $error->getAttributes();
 
@@ -210,7 +235,7 @@ class Emulative extends Lexer {
                 if ($patchType === 'add') {
                     $posDelta += strlen($patchText);
                     $lineDelta += substr_count($patchText, "\n");
-                } elseif ($patchType === 'remove') {
+                } else if ($patchType === 'remove') {
                     $posDelta -= strlen($patchText);
                     $lineDelta -= substr_count($patchText, "\n");
                 }

lib/PhpParser/Lexer/TokenEmulator/AsymmetricVisibilityTokenEmulator.php

@@ -1,93 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\PhpVersion;
-use PhpParser\Token;
-
-final class AsymmetricVisibilityTokenEmulator extends TokenEmulator {
-    public function getPhpVersion(): PhpVersion {
-        return PhpVersion::fromComponents(8, 4);
-    }
-    public function isEmulationNeeded(string $code): bool {
-        $code = strtolower($code);
-        return strpos($code, 'public(set)') !== false ||
-            strpos($code, 'protected(set)') !== false ||
-            strpos($code, 'private(set)') !== false;
-    }
-
-    public function emulate(string $code, array $tokens): array {
-        $map = [
-            \T_PUBLIC => \T_PUBLIC_SET,
-            \T_PROTECTED => \T_PROTECTED_SET,
-            \T_PRIVATE => \T_PRIVATE_SET,
-        ];
-        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            $token = $tokens[$i];
-            if (isset($map[$token->id]) && $i + 3 < $c && $tokens[$i + 1]->text === '(' &&
-                $tokens[$i + 2]->id === \T_STRING && \strtolower($tokens[$i + 2]->text) === 'set' &&
-                $tokens[$i + 3]->text === ')' &&
-                $this->isKeywordContext($tokens, $i)
-            ) {
-                array_splice($tokens, $i, 4, [
-                    new Token(
-                        $map[$token->id], $token->text . '(' . $tokens[$i + 2]->text . ')',
-                        $token->line, $token->pos),
-                ]);
-                $c -= 3;
-            }
-        }
-
-        return $tokens;
-    }
-
-    public function reverseEmulate(string $code, array $tokens): array {
-        $reverseMap = [
-            \T_PUBLIC_SET => \T_PUBLIC,
-            \T_PROTECTED_SET => \T_PROTECTED,
-            \T_PRIVATE_SET => \T_PRIVATE,
-        ];
-        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            $token = $tokens[$i];
-            if (isset($reverseMap[$token->id]) &&
-                \preg_match('/(public|protected|private)\((set)\)/i', $token->text, $matches)
-            ) {
-                [, $modifier, $set] = $matches;
-                $modifierLen = \strlen($modifier);
-                array_splice($tokens, $i, 1, [
-                    new Token($reverseMap[$token->id], $modifier, $token->line, $token->pos),
-                    new Token(\ord('('), '(', $token->line, $token->pos + $modifierLen),
-                    new Token(\T_STRING, $set, $token->line, $token->pos + $modifierLen + 1),
-                    new Token(\ord(')'), ')', $token->line, $token->pos + $modifierLen + 4),
-                ]);
-                $i += 3;
-                $c += 3;
-            }
-        }
-
-        return $tokens;
-    }
-
-    /** @param Token[] $tokens */
-    protected function isKeywordContext(array $tokens, int $pos): bool {
-        $prevToken = $this->getPreviousNonSpaceToken($tokens, $pos);
-        if ($prevToken === null) {
-            return false;
-        }
-        return $prevToken->id !== \T_OBJECT_OPERATOR
-            && $prevToken->id !== \T_NULLSAFE_OBJECT_OPERATOR;
-    }
-
-    /** @param Token[] $tokens */
-    private function getPreviousNonSpaceToken(array $tokens, int $start): ?Token {
-        for ($i = $start - 1; $i >= 0; --$i) {
-            if ($tokens[$i]->id === T_WHITESPACE) {
-                continue;
-            }
-
-            return $tokens[$i];
-        }
-
-        return null;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php

@@ -2,36 +2,43 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
+use PhpParser\Lexer\Emulative;
-use PhpParser\Token;
 
-final class AttributeEmulator extends TokenEmulator {
+final class AttributeEmulator extends TokenEmulator
-    public function getPhpVersion(): PhpVersion {
+{
-        return PhpVersion::fromComponents(8, 0);
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_0;
     }
 
-    public function isEmulationNeeded(string $code): bool {
+    public function isEmulationNeeded(string $code) : bool
+    {
         return strpos($code, '#[') !== false;
     }
 
-    public function emulate(string $code, array $tokens): array {
+    public function emulate(string $code, array $tokens): array
+    {
         // We need to manually iterate and manage a count because we'll change
         // the tokens array on the way.
+        $line = 1;
         for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            $token = $tokens[$i];
+            if ($tokens[$i] === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1] === '[') {
-            if ($token->text === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1]->text === '[') {
                 array_splice($tokens, $i, 2, [
-                    new Token(\T_ATTRIBUTE, '#[', $token->line, $token->pos),
+                    [\T_ATTRIBUTE, '#[', $line]
                 ]);
                 $c--;
                 continue;
             }
+            if (\is_array($tokens[$i])) {
+                $line += substr_count($tokens[$i][1], "\n");
+            }
         }
 
         return $tokens;
     }
 
-    public function reverseEmulate(string $code, array $tokens): array {
+    public function reverseEmulate(string $code, array $tokens): array
+    {
         // TODO
         return $tokens;
     }

lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php

@@ -0,0 +1,47 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class CoaleseEqualTokenEmulator extends TokenEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_7_4;
+    }
+
+    public function isEmulationNeeded(string $code): bool
+    {
+        return strpos($code, '??=') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $line = 1;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            if (isset($tokens[$i + 1])) {
+                if ($tokens[$i][0] === T_COALESCE && $tokens[$i + 1] === '=') {
+                    array_splice($tokens, $i, 2, [
+                        [\T_COALESCE_EQUAL, '??=', $line]
+                    ]);
+                    $c--;
+                    continue;
+                }
+            }
+            if (\is_array($tokens[$i])) {
+                $line += substr_count($tokens[$i][1], "\n");
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        // ??= was not valid code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/EnumTokenEmulator.php

@@ -2,25 +2,30 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
+use PhpParser\Lexer\Emulative;
 
-final class EnumTokenEmulator extends KeywordEmulator {
+final class EnumTokenEmulator extends KeywordEmulator
-    public function getPhpVersion(): PhpVersion {
+{
-        return PhpVersion::fromComponents(8, 1);
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_1;
     }
 
-    public function getKeywordString(): string {
+    public function getKeywordString(): string
+    {
         return 'enum';
     }
 
-    public function getKeywordToken(): int {
+    public function getKeywordToken(): int
+    {
         return \T_ENUM;
     }
 
-    protected function isKeywordContext(array $tokens, int $pos): bool {
+    protected function isKeywordContext(array $tokens, int $pos): bool
+    {
         return parent::isKeywordContext($tokens, $pos)
             && isset($tokens[$pos + 2])
-            && $tokens[$pos + 1]->id === \T_WHITESPACE
+            && $tokens[$pos + 1][0] === \T_WHITESPACE
-            && $tokens[$pos + 2]->id === \T_STRING;
+            && $tokens[$pos + 2][0] === \T_STRING;
     }
-}
+}

lib/PhpParser/Lexer/TokenEmulator/ExplicitOctalEmulator.php

@@ -2,12 +2,11 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
+use PhpParser\Lexer\Emulative;
-use PhpParser\Token;
 
 class ExplicitOctalEmulator extends TokenEmulator {
-    public function getPhpVersion(): PhpVersion {
+    public function getPhpVersion(): string {
-        return PhpVersion::fromComponents(8, 1);
+        return Emulative::PHP_8_1;
     }
 
     public function isEmulationNeeded(string $code): bool {
@@ -16,14 +15,13 @@ class ExplicitOctalEmulator extends TokenEmulator {
 
     public function emulate(string $code, array $tokens): array {
         for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            $token = $tokens[$i];
+            if ($tokens[$i][0] == \T_LNUMBER && $tokens[$i][1] === '0' &&
-            if ($token->id == \T_LNUMBER && $token->text === '0' &&
+                isset($tokens[$i + 1]) && $tokens[$i + 1][0] == \T_STRING &&
-                isset($tokens[$i + 1]) && $tokens[$i + 1]->id == \T_STRING &&
+                preg_match('/[oO][0-7]+(?:_[0-7]+)*/', $tokens[$i + 1][1])
-                preg_match('/[oO][0-7]+(?:_[0-7]+)*/', $tokens[$i + 1]->text)
             ) {
-                $tokenKind = $this->resolveIntegerOrFloatToken($tokens[$i + 1]->text);
+                $tokenKind = $this->resolveIntegerOrFloatToken($tokens[$i + 1][1]);
                 array_splice($tokens, $i, 2, [
-                    new Token($tokenKind, '0' . $tokens[$i + 1]->text, $token->line, $token->pos),
+                    [$tokenKind, '0' . $tokens[$i + 1][1], $tokens[$i][2]],
                 ]);
                 $c--;
             }
@@ -31,7 +29,8 @@ class ExplicitOctalEmulator extends TokenEmulator {
         return $tokens;
     }
 
-    private function resolveIntegerOrFloatToken(string $str): int {
+    private function resolveIntegerOrFloatToken(string $str): int
+    {
         $str = substr($str, 1);
         $str = str_replace('_', '', $str);
         $num = octdec($str);
@@ -42,4 +41,4 @@ class ExplicitOctalEmulator extends TokenEmulator {
         // Explicit octals were not legal code previously, don't bother.
         return $tokens;
     }
-}
+}

lib/PhpParser/Lexer/TokenEmulator/FlexibleDocStringEmulator.php

@@ -0,0 +1,76 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class FlexibleDocStringEmulator extends TokenEmulator
+{
+    const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
+/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
+(?:.*\r?\n)*?
+(?<indentation>\h*)\2(?![a-zA-Z0-9_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
+REGEX;
+
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_7_3;
+    }
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        return strpos($code, '<<<') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // Handled by preprocessing + fixup.
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        // Not supported.
+        return $tokens;
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
+            // No heredoc/nowdoc found
+            return $code;
+        }
+
+        // Keep track of how much we need to adjust string offsets due to the modifications we
+        // already made
+        $posDelta = 0;
+        foreach ($matches as $match) {
+            $indentation = $match['indentation'][0];
+            $indentationStart = $match['indentation'][1];
+
+            $separator = $match['separator'][0];
+            $separatorStart = $match['separator'][1];
+
+            if ($indentation === '' && $separator !== '') {
+                // Ordinary heredoc/nowdoc
+                continue;
+            }
+
+            if ($indentation !== '') {
+                // Remove indentation
+                $indentationLen = strlen($indentation);
+                $code = substr_replace($code, '', $indentationStart + $posDelta, $indentationLen);
+                $patches[] = [$indentationStart + $posDelta, 'add', $indentation];
+                $posDelta -= $indentationLen;
+            }
+
+            if ($separator === '') {
+                // Insert newline as separator
+                $code = substr_replace($code, "\n", $separatorStart + $posDelta, 0);
+                $patches[] = [$separatorStart + $posDelta, 'remove', "\n"];
+                $posDelta += 1;
+            }
+        }
+
+        return $code;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php

@@ -0,0 +1,23 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class FnTokenEmulator extends KeywordEmulator
+{
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_7_4;
+    }
+
+    public function getKeywordString(): string
+    {
+        return 'fn';
+    }
+
+    public function getKeywordToken(): int
+    {
+        return \T_FN;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/KeywordEmulator.php

@@ -2,42 +2,43 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\Token;
+abstract class KeywordEmulator extends TokenEmulator
+{
+    abstract function getKeywordString(): string;
+    abstract function getKeywordToken(): int;
 
-abstract class KeywordEmulator extends TokenEmulator {
+    public function isEmulationNeeded(string $code): bool
-    abstract public function getKeywordString(): string;
+    {
-    abstract public function getKeywordToken(): int;
-
-    public function isEmulationNeeded(string $code): bool {
         return strpos(strtolower($code), $this->getKeywordString()) !== false;
     }
 
-    /** @param Token[] $tokens */
+    protected function isKeywordContext(array $tokens, int $pos): bool
-    protected function isKeywordContext(array $tokens, int $pos): bool {
+    {
-        $prevToken = $this->getPreviousNonSpaceToken($tokens, $pos);
+        $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $pos);
-        if ($prevToken === null) {
+        return $previousNonSpaceToken === null || $previousNonSpaceToken[0] !== \T_OBJECT_OPERATOR;
-            return false;
-        }
-        return $prevToken->id !== \T_OBJECT_OPERATOR
-            && $prevToken->id !== \T_NULLSAFE_OBJECT_OPERATOR;
     }
 
-    public function emulate(string $code, array $tokens): array {
+    public function emulate(string $code, array $tokens): array
+    {
         $keywordString = $this->getKeywordString();
         foreach ($tokens as $i => $token) {
-            if ($token->id === T_STRING && strtolower($token->text) === $keywordString
+            if ($token[0] === T_STRING && strtolower($token[1]) === $keywordString
                     && $this->isKeywordContext($tokens, $i)) {
-                $token->id = $this->getKeywordToken();
+                $tokens[$i][0] = $this->getKeywordToken();
             }
         }
 
         return $tokens;
     }
 
-    /** @param Token[] $tokens */
+    /**
-    private function getPreviousNonSpaceToken(array $tokens, int $start): ?Token {
+     * @param mixed[] $tokens
+     * @return array|string|null
+     */
+    private function getPreviousNonSpaceToken(array $tokens, int $start)
+    {
         for ($i = $start - 1; $i >= 0; --$i) {
-            if ($tokens[$i]->id === T_WHITESPACE) {
+            if ($tokens[$i][0] === T_WHITESPACE) {
                 continue;
             }
 
@@ -47,11 +48,12 @@ abstract class KeywordEmulator extends TokenEmulator {
         return null;
     }
 
-    public function reverseEmulate(string $code, array $tokens): array {
+    public function reverseEmulate(string $code, array $tokens): array
+    {
         $keywordToken = $this->getKeywordToken();
-        foreach ($tokens as $token) {
+        foreach ($tokens as $i => $token) {
-            if ($token->id === $keywordToken) {
+            if ($token[0] === $keywordToken) {
-                $token->id = \T_STRING;
+                $tokens[$i][0] = \T_STRING;
             }
         }
 

lib/PhpParser/Lexer/TokenEmulator/MatchTokenEmulator.php

@@ -2,18 +2,22 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
+use PhpParser\Lexer\Emulative;
 
-final class MatchTokenEmulator extends KeywordEmulator {
+final class MatchTokenEmulator extends KeywordEmulator
-    public function getPhpVersion(): PhpVersion {
+{
-        return PhpVersion::fromComponents(8, 0);
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_0;
     }
 
-    public function getKeywordString(): string {
+    public function getKeywordString(): string
+    {
         return 'match';
     }
 
-    public function getKeywordToken(): int {
+    public function getKeywordToken(): int
+    {
         return \T_MATCH;
     }
 }

lib/PhpParser/Lexer/TokenEmulator/NullsafeTokenEmulator.php

@@ -2,58 +2,65 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
+use PhpParser\Lexer\Emulative;
-use PhpParser\Token;
 
-final class NullsafeTokenEmulator extends TokenEmulator {
+final class NullsafeTokenEmulator extends TokenEmulator
-    public function getPhpVersion(): PhpVersion {
+{
-        return PhpVersion::fromComponents(8, 0);
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_0;
     }
 
-    public function isEmulationNeeded(string $code): bool {
+    public function isEmulationNeeded(string $code): bool
+    {
         return strpos($code, '?->') !== false;
     }
 
-    public function emulate(string $code, array $tokens): array {
+    public function emulate(string $code, array $tokens): array
+    {
         // We need to manually iterate and manage a count because we'll change
         // the tokens array on the way
+        $line = 1;
         for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
-            $token = $tokens[$i];
+            if ($tokens[$i] === '?' && isset($tokens[$i + 1]) && $tokens[$i + 1][0] === \T_OBJECT_OPERATOR) {
-            if ($token->text === '?' && isset($tokens[$i + 1]) && $tokens[$i + 1]->id === \T_OBJECT_OPERATOR) {
                 array_splice($tokens, $i, 2, [
-                    new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', $token->line, $token->pos),
+                    [\T_NULLSAFE_OBJECT_OPERATOR, '?->', $line]
                 ]);
                 $c--;
                 continue;
             }
 
             // Handle ?-> inside encapsed string.
-            if ($token->id === \T_ENCAPSED_AND_WHITESPACE && isset($tokens[$i - 1])
+            if ($tokens[$i][0] === \T_ENCAPSED_AND_WHITESPACE && isset($tokens[$i - 1])
-                && $tokens[$i - 1]->id === \T_VARIABLE
+                && $tokens[$i - 1][0] === \T_VARIABLE
-                && preg_match('/^\?->([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)/', $token->text, $matches)
+                && preg_match('/^\?->([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)/', $tokens[$i][1], $matches)
             ) {
                 $replacement = [
-                    new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', $token->line, $token->pos),
+                    [\T_NULLSAFE_OBJECT_OPERATOR, '?->', $line],
-                    new Token(\T_STRING, $matches[1], $token->line, $token->pos + 3),
+                    [\T_STRING, $matches[1], $line],
                 ];
-                $matchLen = \strlen($matches[0]);
+                if (\strlen($matches[0]) !== \strlen($tokens[$i][1])) {
-                if ($matchLen !== \strlen($token->text)) {
+                    $replacement[] = [
-                    $replacement[] = new Token(
                         \T_ENCAPSED_AND_WHITESPACE,
-                        \substr($token->text, $matchLen),
+                        \substr($tokens[$i][1], \strlen($matches[0])),
-                        $token->line, $token->pos + $matchLen
+                        $line
-                    );
+                    ];
                 }
                 array_splice($tokens, $i, 1, $replacement);
                 $c += \count($replacement) - 1;
                 continue;
             }
+
+            if (\is_array($tokens[$i])) {
+                $line += substr_count($tokens[$i][1], "\n");
+            }
         }
 
         return $tokens;
     }
 
-    public function reverseEmulate(string $code, array $tokens): array {
+    public function reverseEmulate(string $code, array $tokens): array
+    {
         // ?-> was not valid code previously, don't bother.
         return $tokens;
     }

lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -0,0 +1,105 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class NumericLiteralSeparatorEmulator extends TokenEmulator
+{
+    const BIN = '(?:0b[01]+(?:_[01]+)*)';
+    const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
+    const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
+    const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
+    const EXP = '(?:e[+-]?' . self::DEC . ')';
+    const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
+    const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
+
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_7_4;
+    }
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        return preg_match('~[0-9]_[0-9]~', $code)
+            || preg_match('~0x[0-9a-f]+_[0-9a-f]~i', $code);
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $codeOffset = 0;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            $tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
+
+            if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
+            assert($res, "No number at number token position");
+
+            $match = $matches[0];
+            $matchLen = \strlen($match);
+            if ($matchLen === $tokenLen) {
+                // Original token already holds the full number.
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $tokenKind = $this->resolveIntegerOrFloatToken($match);
+            $newTokens = [[$tokenKind, $match, $token[2]]];
+
+            $numTokens = 1;
+            $len = $tokenLen;
+            while ($matchLen > $len) {
+                $nextToken = $tokens[$i + $numTokens];
+                $nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
+                $nextTokenLen = \strlen($nextTokenText);
+
+                $numTokens++;
+                if ($matchLen < $len + $nextTokenLen) {
+                    // Split trailing characters into a partial token.
+                    assert(is_array($nextToken), "Partial token should be an array token");
+                    $partialText = substr($nextTokenText, $matchLen - $len);
+                    $newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
+                    break;
+                }
+
+                $len += $nextTokenLen;
+            }
+
+            array_splice($tokens, $i, $numTokens, $newTokens);
+            $c -= $numTokens - \count($newTokens);
+            $codeOffset += $matchLen;
+        }
+
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int
+    {
+        $str = str_replace('_', '', $str);
+
+        if (stripos($str, '0b') === 0) {
+            $num = bindec($str);
+        } elseif (stripos($str, '0x') === 0) {
+            $num = hexdec($str);
+        } elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
+            $num = octdec($str);
+        } else {
+            $num = +$str;
+        }
+
+        return is_float($num) ? T_DNUMBER : T_LNUMBER;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array
+    {
+        // Numeric separators were not legal code previously, don't bother.
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/PropertyTokenEmulator.php

@@ -1,19 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Lexer\TokenEmulator;
-
-use PhpParser\PhpVersion;
-
-final class PropertyTokenEmulator extends KeywordEmulator {
-    public function getPhpVersion(): PhpVersion {
-        return PhpVersion::fromComponents(8, 4);
-    }
-
-    public function getKeywordString(): string {
-        return '__property__';
-    }
-
-    public function getKeywordToken(): int {
-        return \T_PROPERTY_C;
-    }
-}

lib/PhpParser/Lexer/TokenEmulator/ReadonlyFunctionTokenEmulator.php

@@ -2,7 +2,7 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
+use PhpParser\Lexer\Emulative;
 
 /*
  * In PHP 8.1, "readonly(" was special cased in the lexer in order to support functions with
@@ -20,8 +20,8 @@ class ReadonlyFunctionTokenEmulator extends KeywordEmulator {
         return \T_READONLY;
     }
 
-    public function getPhpVersion(): PhpVersion {
+    public function getPhpVersion(): string {
-        return PhpVersion::fromComponents(8, 2);
+        return Emulative::PHP_8_2;
     }
 
     public function reverseEmulate(string $code, array $tokens): array {

lib/PhpParser/Lexer/TokenEmulator/ReadonlyTokenEmulator.php

@@ -2,30 +2,35 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
+use PhpParser\Lexer\Emulative;
 
-final class ReadonlyTokenEmulator extends KeywordEmulator {
+final class ReadonlyTokenEmulator extends KeywordEmulator
-    public function getPhpVersion(): PhpVersion {
+{
-        return PhpVersion::fromComponents(8, 1);
+    public function getPhpVersion(): string
+    {
+        return Emulative::PHP_8_1;
     }
 
-    public function getKeywordString(): string {
+    public function getKeywordString(): string
+    {
         return 'readonly';
     }
 
-    public function getKeywordToken(): int {
+    public function getKeywordToken(): int
+    {
         return \T_READONLY;
     }
 
-    protected function isKeywordContext(array $tokens, int $pos): bool {
+    protected function isKeywordContext(array $tokens, int $pos): bool
+    {
         if (!parent::isKeywordContext($tokens, $pos)) {
             return false;
         }
         // Support "function readonly("
         return !(isset($tokens[$pos + 1]) &&
-                 ($tokens[$pos + 1]->text === '(' ||
+                 ($tokens[$pos + 1][0] === '(' ||
-                  ($tokens[$pos + 1]->id === \T_WHITESPACE &&
+                  ($tokens[$pos + 1][0] === \T_WHITESPACE &&
                    isset($tokens[$pos + 2]) &&
-                   $tokens[$pos + 2]->text === '(')));
+                   $tokens[$pos + 2][0] === '(')));
     }
 }

lib/PhpParser/Lexer/TokenEmulator/ReverseEmulator.php

@@ -2,20 +2,19 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
-
 /**
  * Reverses emulation direction of the inner emulator.
  */
-final class ReverseEmulator extends TokenEmulator {
+final class ReverseEmulator extends TokenEmulator
+{
     /** @var TokenEmulator Inner emulator */
-    private TokenEmulator $emulator;
+    private $emulator;
 
     public function __construct(TokenEmulator $emulator) {
         $this->emulator = $emulator;
     }
 
-    public function getPhpVersion(): PhpVersion {
+    public function getPhpVersion(): string {
         return $this->emulator->getPhpVersion();
     }
 
@@ -34,4 +33,4 @@ final class ReverseEmulator extends TokenEmulator {
     public function preprocessCode(string $code, array &$patches): string {
         return $code;
     }
-}
+}

lib/PhpParser/Lexer/TokenEmulator/TokenEmulator.php

@@ -2,28 +2,23 @@
 
 namespace PhpParser\Lexer\TokenEmulator;
 
-use PhpParser\PhpVersion;
-use PhpParser\Token;
-
 /** @internal */
-abstract class TokenEmulator {
+abstract class TokenEmulator
-    abstract public function getPhpVersion(): PhpVersion;
+{
+    abstract public function getPhpVersion(): string;
 
     abstract public function isEmulationNeeded(string $code): bool;
 
     /**
-     * @param Token[] $tokens Original tokens
+     * @return array Modified Tokens
-     * @return Token[] Modified Tokens
      */
     abstract public function emulate(string $code, array $tokens): array;
 
     /**
-     * @param Token[] $tokens Original tokens
+     * @return array Modified Tokens
-     * @return Token[] Modified Tokens
      */
     abstract public function reverseEmulate(string $code, array $tokens): array;
 
-    /** @param array{int, string, string}[] $patches */
     public function preprocessCode(string $code, array &$patches): string {
         return $code;
     }

lib/PhpParser/Modifiers.php

@@ -1,85 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser;
-
-/**
- * Modifiers used (as a bit mask) by various flags subnodes, for example on classes, functions,
- * properties and constants.
- */
-final class Modifiers {
-    public const PUBLIC    =  1;
-    public const PROTECTED =  2;
-    public const PRIVATE   =  4;
-    public const STATIC    =  8;
-    public const ABSTRACT  = 16;
-    public const FINAL     = 32;
-    public const READONLY  = 64;
-    public const PUBLIC_SET = 128;
-    public const PROTECTED_SET = 256;
-    public const PRIVATE_SET = 512;
-
-    public const VISIBILITY_MASK = self::PUBLIC | self::PROTECTED | self::PRIVATE;
-
-    public const VISIBILITY_SET_MASK = self::PUBLIC_SET | self::PROTECTED_SET | self::PRIVATE_SET;
-
-    private const TO_STRING_MAP = [
-        self::PUBLIC  => 'public',
-        self::PROTECTED => 'protected',
-        self::PRIVATE => 'private',
-        self::STATIC  => 'static',
-        self::ABSTRACT => 'abstract',
-        self::FINAL  => 'final',
-        self::READONLY  => 'readonly',
-        self::PUBLIC_SET => 'public(set)',
-        self::PROTECTED_SET => 'protected(set)',
-        self::PRIVATE_SET => 'private(set)',
-    ];
-
-    public static function toString(int $modifier): string {
-        if (!isset(self::TO_STRING_MAP[$modifier])) {
-            throw new \InvalidArgumentException("Unknown modifier $modifier");
-        }
-        return self::TO_STRING_MAP[$modifier];
-    }
-
-    private static function isValidModifier(int $modifier): bool {
-        $isPow2 = ($modifier & ($modifier - 1)) == 0 && $modifier != 0;
-        return $isPow2 && $modifier <= self::PRIVATE_SET;
-    }
-
-    /**
-     * @internal
-     */
-    public static function verifyClassModifier(int $a, int $b): void {
-        assert(self::isValidModifier($b));
-        if (($a & $b) != 0) {
-            throw new Error(
-                'Multiple ' . self::toString($b) . ' modifiers are not allowed');
-        }
-
-        if ($a & 48 && $b & 48) {
-            throw new Error('Cannot use the final modifier on an abstract class');
-        }
-    }
-
-    /**
-     * @internal
-     */
-    public static function verifyModifier(int $a, int $b): void {
-        assert(self::isValidModifier($b));
-        if (($a & Modifiers::VISIBILITY_MASK && $b & Modifiers::VISIBILITY_MASK) ||
-            ($a & Modifiers::VISIBILITY_SET_MASK && $b & Modifiers::VISIBILITY_SET_MASK)
-        ) {
-            throw new Error('Multiple access type modifiers are not allowed');
-        }
-
-        if (($a & $b) != 0) {
-            throw new Error(
-                'Multiple ' . self::toString($b) . ' modifiers are not allowed');
-        }
-
-        if ($a & 48 && $b & 48) {
-            throw new Error('Cannot use the final modifier on an abstract class member');
-        }
-    }
-}

lib/PhpParser/NameContext.php

@@ -6,18 +6,19 @@ use PhpParser\Node\Name;
 use PhpParser\Node\Name\FullyQualified;
 use PhpParser\Node\Stmt;
 
-class NameContext {
+class NameContext
+{
     /** @var null|Name Current namespace */
-    protected ?Name $namespace;
+    protected $namespace;
 
     /** @var Name[][] Map of format [aliasType => [aliasName => originalName]] */
-    protected array $aliases = [];
+    protected $aliases = [];
 
     /** @var Name[][] Same as $aliases but preserving original case */
-    protected array $origAliases = [];
+    protected $origAliases = [];
 
     /** @var ErrorHandler Error handler */
-    protected ErrorHandler $errorHandler;
+    protected $errorHandler;
 
     /**
      * Create a name context.
@@ -35,7 +36,7 @@ class NameContext {
      *
      * @param Name|null $namespace Null is the global namespace
      */
-    public function startNamespace(?Name $namespace = null): void {
+    public function startNamespace(?Name $namespace = null) {
         $this->namespace = $namespace;
         $this->origAliases = $this->aliases = [
             Stmt\Use_::TYPE_NORMAL   => [],
@@ -47,12 +48,12 @@ class NameContext {
     /**
      * Add an alias / import.
      *
-     * @param Name $name Original name
+     * @param Name   $name        Original name
-     * @param string $aliasName Aliased name
+     * @param string $aliasName   Aliased name
-     * @param Stmt\Use_::TYPE_* $type One of Stmt\Use_::TYPE_*
+     * @param int    $type        One of Stmt\Use_::TYPE_*
-     * @param array<string, mixed> $errorAttrs Attributes to use to report an error
+     * @param array  $errorAttrs Attributes to use to report an error
      */
-    public function addAlias(Name $name, string $aliasName, int $type, array $errorAttrs = []): void {
+    public function addAlias(Name $name, string $aliasName, int $type, array $errorAttrs = []) {
         // Constant names are case sensitive, everything else case insensitive
         if ($type === Stmt\Use_::TYPE_CONSTANT) {
             $aliasLookupName = $aliasName;
@@ -86,7 +87,7 @@ class NameContext {
      *
      * @return null|Name Namespace (or null if global namespace)
      */
-    public function getNamespace(): ?Name {
+    public function getNamespace() {
         return $this->namespace;
     }
 
@@ -94,11 +95,11 @@ class NameContext {
      * Get resolved name.
      *
      * @param Name $name Name to resolve
-     * @param Stmt\Use_::TYPE_* $type One of Stmt\Use_::TYPE_{FUNCTION|CONSTANT}
+     * @param int  $type One of Stmt\Use_::TYPE_{FUNCTION|CONSTANT}
      *
      * @return null|Name Resolved name, or null if static resolution is not possible
      */
-    public function getResolvedName(Name $name, int $type): ?Name {
+    public function getResolvedName(Name $name, int $type) {
         // don't resolve special class names
         if ($type === Stmt\Use_::TYPE_NORMAL && $name->isSpecialClassName()) {
             if (!$name->isUnqualified()) {
@@ -141,7 +142,7 @@ class NameContext {
      *
      * @return Name Resolved name
      */
-    public function getResolvedClassName(Name $name): Name {
+    public function getResolvedClassName(Name $name) : Name {
         return $this->getResolvedName($name, Stmt\Use_::TYPE_NORMAL);
     }
 
@@ -149,11 +150,11 @@ class NameContext {
      * Get possible ways of writing a fully qualified name (e.g., by making use of aliases).
      *
      * @param string $name Fully-qualified name (without leading namespace separator)
-     * @param Stmt\Use_::TYPE_* $type One of Stmt\Use_::TYPE_*
+     * @param int    $type One of Stmt\Use_::TYPE_*
      *
      * @return Name[] Possible representations of the name
      */
-    public function getPossibleNames(string $name, int $type): array {
+    public function getPossibleNames(string $name, int $type) : array {
         $lcName = strtolower($name);
 
         if ($type === Stmt\Use_::TYPE_NORMAL) {
@@ -185,7 +186,7 @@ class NameContext {
         // Check for relevant type-specific use statements
         foreach ($this->origAliases[$type] as $alias => $orig) {
             if ($type === Stmt\Use_::TYPE_CONSTANT) {
-                // Constants are complicated-sensitive
+                // Constants are are complicated-sensitive
                 $normalizedOrig = $this->normalizeConstName($orig->toString());
                 if ($normalizedOrig === $this->normalizeConstName($name)) {
                     $possibleNames[] = new Name($alias);
@@ -205,11 +206,11 @@ class NameContext {
      * Get shortest representation of this fully-qualified name.
      *
      * @param string $name Fully-qualified name (without leading namespace separator)
-     * @param Stmt\Use_::TYPE_* $type One of Stmt\Use_::TYPE_*
+     * @param int    $type One of Stmt\Use_::TYPE_*
      *
      * @return Name Shortest representation
      */
-    public function getShortName(string $name, int $type): Name {
+    public function getShortName(string $name, int $type) : Name {
         $possibleNames = $this->getPossibleNames($name, $type);
 
         // Find shortest name
@@ -223,10 +224,10 @@ class NameContext {
             }
         }
 
-        return $shortestName;
+       return $shortestName;
     }
 
-    private function resolveAlias(Name $name, int $type): ?FullyQualified {
+    private function resolveAlias(Name $name, $type) {
         $firstPart = $name->getFirst();
 
         if ($name->isQualified()) {
@@ -249,7 +250,7 @@ class NameContext {
         return null;
     }
 
-    private function getNamespaceRelativeName(string $name, string $lcName, int $type): ?Name {
+    private function getNamespaceRelativeName(string $name, string $lcName, int $type) {
         if (null === $this->namespace) {
             return new Name($name);
         }
@@ -270,7 +271,7 @@ class NameContext {
         return null;
     }
 
-    private function normalizeConstName(string $name): string {
+    private function normalizeConstName(string $name) {
         $nsSep = strrpos($name, '\\');
         if (false === $nsSep) {
             return $name;

lib/PhpParser/Node.php

@@ -2,31 +2,28 @@
 
 namespace PhpParser;
 
-interface Node {
+interface Node
+{
     /**
      * Gets the type of the node.
      *
-     * @psalm-return non-empty-string
      * @return string Type of the node
      */
-    public function getType(): string;
+    public function getType() : string;
 
     /**
      * Gets the names of the sub nodes.
      *
-     * @return string[] Names of sub nodes
+     * @return array Names of sub nodes
      */
-    public function getSubNodeNames(): array;
+    public function getSubNodeNames() : array;
 
     /**
      * Gets line the node started in (alias of getStartLine).
      *
      * @return int Start line (or -1 if not available)
-     * @phpstan-return -1|positive-int
-     *
-     * @deprecated Use getStartLine() instead
      */
-    public function getLine(): int;
+    public function getLine() : int;
 
     /**
      * Gets line the node started in.
@@ -34,9 +31,8 @@ interface Node {
      * Requires the 'startLine' attribute to be enabled in the lexer (enabled by default).
      *
      * @return int Start line (or -1 if not available)
-     * @phpstan-return -1|positive-int
      */
-    public function getStartLine(): int;
+    public function getStartLine() : int;
 
     /**
      * Gets the line the node ended in.
@@ -44,9 +40,8 @@ interface Node {
      * Requires the 'endLine' attribute to be enabled in the lexer (enabled by default).
      *
      * @return int End line (or -1 if not available)
-     * @phpstan-return -1|positive-int
      */
-    public function getEndLine(): int;
+    public function getEndLine() : int;
 
     /**
      * Gets the token offset of the first token that is part of this node.
@@ -57,7 +52,7 @@ interface Node {
      *
      * @return int Token start position (or -1 if not available)
      */
-    public function getStartTokenPos(): int;
+    public function getStartTokenPos() : int;
 
     /**
      * Gets the token offset of the last token that is part of this node.
@@ -68,7 +63,7 @@ interface Node {
      *
      * @return int Token end position (or -1 if not available)
      */
-    public function getEndTokenPos(): int;
+    public function getEndTokenPos() : int;
 
     /**
      * Gets the file offset of the first character that is part of this node.
@@ -77,7 +72,7 @@ interface Node {
      *
      * @return int File start position (or -1 if not available)
      */
-    public function getStartFilePos(): int;
+    public function getStartFilePos() : int;
 
     /**
      * Gets the file offset of the last character that is part of this node.
@@ -86,7 +81,7 @@ interface Node {
      *
      * @return int File end position (or -1 if not available)
      */
-    public function getEndFilePos(): int;
+    public function getEndFilePos() : int;
 
     /**
      * Gets all comments directly preceding this node.
@@ -95,14 +90,14 @@ interface Node {
      *
      * @return Comment[]
      */
-    public function getComments(): array;
+    public function getComments() : array;
 
     /**
      * Gets the doc comment of the node.
      *
      * @return null|Comment\Doc Doc comment object or null
      */
-    public function getDocComment(): ?Comment\Doc;
+    public function getDocComment();
 
     /**
      * Sets the doc comment of the node.
@@ -111,24 +106,30 @@ interface Node {
      *
      * @param Comment\Doc $docComment Doc comment to set
      */
-    public function setDocComment(Comment\Doc $docComment): void;
+    public function setDocComment(Comment\Doc $docComment);
 
     /**
      * Sets an attribute on a node.
      *
-     * @param mixed $value
+     * @param string $key
+     * @param mixed  $value
      */
-    public function setAttribute(string $key, $value): void;
+    public function setAttribute(string $key, $value);
 
     /**
      * Returns whether an attribute exists.
+     *
+     * @param string $key
+     *
+     * @return bool
      */
-    public function hasAttribute(string $key): bool;
+    public function hasAttribute(string $key) : bool;
 
     /**
      * Returns the value of an attribute.
      *
-     * @param mixed $default
+     * @param string $key
+     * @param mixed  $default
      *
      * @return mixed
      */
@@ -137,14 +138,14 @@ interface Node {
     /**
      * Returns all the attributes of this node.
      *
-     * @return array<string, mixed>
+     * @return array
      */
-    public function getAttributes(): array;
+    public function getAttributes() : array;
 
     /**
      * Replaces all the attributes of this node.
      *
-     * @param array<string, mixed> $attributes
+     * @param array $attributes
      */
-    public function setAttributes(array $attributes): void;
+    public function setAttributes(array $attributes);
 }

lib/PhpParser/Node/Arg.php

@@ -2,25 +2,27 @@
 
 namespace PhpParser\Node;
 
+use PhpParser\Node\VariadicPlaceholder;
 use PhpParser\NodeAbstract;
 
-class Arg extends NodeAbstract {
+class Arg extends NodeAbstract
+{
     /** @var Identifier|null Parameter name (for named parameters) */
-    public ?Identifier $name;
+    public $name;
     /** @var Expr Value to pass */
-    public Expr $value;
+    public $value;
     /** @var bool Whether to pass by ref */
-    public bool $byRef;
+    public $byRef;
     /** @var bool Whether to unpack the argument */
-    public bool $unpack;
+    public $unpack;
 
     /**
      * Constructs a function call argument node.
      *
-     * @param Expr $value Value to pass
+     * @param Expr  $value      Value to pass
-     * @param bool $byRef Whether to pass by ref
+     * @param bool  $byRef      Whether to pass by ref
-     * @param bool $unpack Whether to unpack the argument
+     * @param bool  $unpack     Whether to unpack the argument
-     * @param array<string, mixed> $attributes Additional attributes
+     * @param array $attributes Additional attributes
      * @param Identifier|null $name Parameter name (for named parameters)
      */
     public function __construct(
@@ -34,11 +36,11 @@ class Arg extends NodeAbstract {
         $this->unpack = $unpack;
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['name', 'value', 'byRef', 'unpack'];
     }
-
+    
-    public function getType(): string {
+    public function getType() : string {
         return 'Arg';
     }
 }

lib/PhpParser/Node/ArrayItem.php

@@ -1,43 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-use PhpParser\NodeAbstract;
-
-class ArrayItem extends NodeAbstract {
-    /** @var null|Expr Key */
-    public ?Expr $key;
-    /** @var Expr Value */
-    public Expr $value;
-    /** @var bool Whether to assign by reference */
-    public bool $byRef;
-    /** @var bool Whether to unpack the argument */
-    public bool $unpack;
-
-    /**
-     * Constructs an array item node.
-     *
-     * @param Expr $value Value
-     * @param null|Expr $key Key
-     * @param bool $byRef Whether to assign by reference
-     * @param array<string, mixed> $attributes Additional attributes
-     */
-    public function __construct(Expr $value, ?Expr $key = null, bool $byRef = false, array $attributes = [], bool $unpack = false) {
-        $this->attributes = $attributes;
-        $this->key = $key;
-        $this->value = $value;
-        $this->byRef = $byRef;
-        $this->unpack = $unpack;
-    }
-
-    public function getSubNodeNames(): array {
-        return ['key', 'value', 'byRef', 'unpack'];
-    }
-
-    public function getType(): string {
-        return 'ArrayItem';
-    }
-}
-
-// @deprecated compatibility alias
-class_alias(ArrayItem::class, Expr\ArrayItem::class);

lib/PhpParser/Node/Attribute.php

@@ -5,17 +5,18 @@ namespace PhpParser\Node;
 use PhpParser\Node;
 use PhpParser\NodeAbstract;
 
-class Attribute extends NodeAbstract {
+class Attribute extends NodeAbstract
+{
     /** @var Name Attribute name */
-    public Name $name;
+    public $name;
 
-    /** @var list<Arg> Attribute arguments */
+    /** @var Arg[] Attribute arguments */
-    public array $args;
+    public $args;
 
     /**
-     * @param Node\Name $name Attribute name
+     * @param Node\Name $name       Attribute name
-     * @param list<Arg> $args Attribute arguments
+     * @param Arg[]     $args       Attribute arguments
-     * @param array<string, mixed> $attributes Additional node attributes
+     * @param array     $attributes Additional node attributes
      */
     public function __construct(Name $name, array $args = [], array $attributes = []) {
         $this->attributes = $attributes;
@@ -23,11 +24,11 @@ class Attribute extends NodeAbstract {
         $this->args = $args;
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['name', 'args'];
     }
 
-    public function getType(): string {
+    public function getType() : string {
         return 'Attribute';
     }
 }

lib/PhpParser/Node/AttributeGroup.php

@@ -2,26 +2,28 @@
 
 namespace PhpParser\Node;
 
+use PhpParser\Node;
 use PhpParser\NodeAbstract;
 
-class AttributeGroup extends NodeAbstract {
+class AttributeGroup extends NodeAbstract
+{
     /** @var Attribute[] Attributes */
-    public array $attrs;
+    public $attrs;
 
     /**
      * @param Attribute[] $attrs PHP attributes
-     * @param array<string, mixed> $attributes Additional node attributes
+     * @param array $attributes Additional node attributes
      */
     public function __construct(array $attrs, array $attributes = []) {
         $this->attributes = $attributes;
         $this->attrs = $attrs;
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['attrs'];
     }
 
-    public function getType(): string {
+    public function getType() : string {
         return 'AttributeGroup';
     }
 }

lib/PhpParser/Node/ClosureUse.php

@@ -1,36 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-use PhpParser\NodeAbstract;
-
-class ClosureUse extends NodeAbstract {
-    /** @var Expr\Variable Variable to use */
-    public Expr\Variable $var;
-    /** @var bool Whether to use by reference */
-    public bool $byRef;
-
-    /**
-     * Constructs a closure use node.
-     *
-     * @param Expr\Variable $var Variable to use
-     * @param bool $byRef Whether to use by reference
-     * @param array<string, mixed> $attributes Additional attributes
-     */
-    public function __construct(Expr\Variable $var, bool $byRef = false, array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->var = $var;
-        $this->byRef = $byRef;
-    }
-
-    public function getSubNodeNames(): array {
-        return ['var', 'byRef'];
-    }
-
-    public function getType(): string {
-        return 'ClosureUse';
-    }
-}
-
-// @deprecated compatibility alias
-class_alias(ClosureUse::class, Expr\ClosureUse::class);

lib/PhpParser/Node/ComplexType.php

@@ -9,5 +9,6 @@ use PhpParser\NodeAbstract;
  *
  * It does not provide any shared behavior and exists only for type-checking purposes.
  */
-abstract class ComplexType extends NodeAbstract {
+abstract class ComplexType extends NodeAbstract
+{
 }

lib/PhpParser/Node/Const_.php

@@ -4,21 +4,22 @@ namespace PhpParser\Node;
 
 use PhpParser\NodeAbstract;
 
-class Const_ extends NodeAbstract {
+class Const_ extends NodeAbstract
+{
     /** @var Identifier Name */
-    public Identifier $name;
+    public $name;
     /** @var Expr Value */
-    public Expr $value;
+    public $value;
 
     /** @var Name|null Namespaced name (if using NameResolver) */
-    public ?Name $namespacedName;
+    public $namespacedName;
 
     /**
      * Constructs a const node for use in class const and const statements.
      *
-     * @param string|Identifier $name Name
+     * @param string|Identifier $name       Name
-     * @param Expr $value Value
+     * @param Expr              $value      Value
-     * @param array<string, mixed> $attributes Additional attributes
+     * @param array             $attributes Additional attributes
      */
     public function __construct($name, Expr $value, array $attributes = []) {
         $this->attributes = $attributes;
@@ -26,11 +27,11 @@ class Const_ extends NodeAbstract {
         $this->value = $value;
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['name', 'value'];
     }
 
-    public function getType(): string {
+    public function getType() : string {
         return 'Const';
     }
 }

lib/PhpParser/Node/DeclareItem.php

@@ -1,37 +0,0 @@
-<?php declare(strict_types=1);
-
-namespace PhpParser\Node;
-
-use PhpParser\Node;
-use PhpParser\NodeAbstract;
-
-class DeclareItem extends NodeAbstract {
-    /** @var Node\Identifier Key */
-    public Identifier $key;
-    /** @var Node\Expr Value */
-    public Expr $value;
-
-    /**
-     * Constructs a declare key=>value pair node.
-     *
-     * @param string|Node\Identifier $key Key
-     * @param Node\Expr $value Value
-     * @param array<string, mixed> $attributes Additional attributes
-     */
-    public function __construct($key, Node\Expr $value, array $attributes = []) {
-        $this->attributes = $attributes;
-        $this->key = \is_string($key) ? new Node\Identifier($key) : $key;
-        $this->value = $value;
-    }
-
-    public function getSubNodeNames(): array {
-        return ['key', 'value'];
-    }
-
-    public function getType(): string {
-        return 'DeclareItem';
-    }
-}
-
-// @deprecated compatibility alias
-class_alias(DeclareItem::class, Stmt\DeclareDeclare::class);

lib/PhpParser/Node/Expr.php

@@ -4,5 +4,6 @@ namespace PhpParser\Node;
 
 use PhpParser\NodeAbstract;
 
-abstract class Expr extends NodeAbstract {
+abstract class Expr extends NodeAbstract
+{
 }

lib/PhpParser/Node/Expr/ArrayDimFetch.php

@@ -4,18 +4,19 @@ namespace PhpParser\Node\Expr;
 
 use PhpParser\Node\Expr;
 
-class ArrayDimFetch extends Expr {
+class ArrayDimFetch extends Expr
+{
     /** @var Expr Variable */
-    public Expr $var;
+    public $var;
     /** @var null|Expr Array index / dim */
-    public ?Expr $dim;
+    public $dim;
 
     /**
      * Constructs an array index fetch node.
      *
-     * @param Expr $var Variable
+     * @param Expr      $var        Variable
-     * @param null|Expr $dim Array index / dim
+     * @param null|Expr $dim        Array index / dim
-     * @param array<string, mixed> $attributes Additional attributes
+     * @param array     $attributes Additional attributes
      */
     public function __construct(Expr $var, ?Expr $dim = null, array $attributes = []) {
         $this->attributes = $attributes;
@@ -23,11 +24,11 @@ class ArrayDimFetch extends Expr {
         $this->dim = $dim;
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['var', 'dim'];
     }
-
+    
-    public function getType(): string {
+    public function getType() : string {
         return 'Expr_ArrayDimFetch';
     }
 }

lib/PhpParser/Node/Expr/ArrayItem.php

@@ -2,10 +2,40 @@
 
 namespace PhpParser\Node\Expr;
 
-require __DIR__ . '/../ArrayItem.php';
+use PhpParser\Node\Expr;
 
-if (false) {
+class ArrayItem extends Expr
-    // For classmap-authoritative support.
+{
-    class ArrayItem extends \PhpParser\Node\ArrayItem {
+    /** @var null|Expr Key */
+    public $key;
+    /** @var Expr Value */
+    public $value;
+    /** @var bool Whether to assign by reference */
+    public $byRef;
+    /** @var bool Whether to unpack the argument */
+    public $unpack;
+
+    /**
+     * Constructs an array item node.
+     *
+     * @param Expr      $value      Value
+     * @param null|Expr $key        Key
+     * @param bool      $byRef      Whether to assign by reference
+     * @param array     $attributes Additional attributes
+     */
+    public function __construct(Expr $value, ?Expr $key = null, bool $byRef = false, array $attributes = [], bool $unpack = false) {
+        $this->attributes = $attributes;
+        $this->key = $key;
+        $this->value = $value;
+        $this->byRef = $byRef;
+        $this->unpack = $unpack;
+    }
+
+    public function getSubNodeNames() : array {
+        return ['key', 'value', 'byRef', 'unpack'];
+    }
+
+    public function getType() : string {
+        return 'Expr_ArrayItem';
     }
 }

lib/PhpParser/Node/Expr/Array_.php

@@ -2,33 +2,33 @@
 
 namespace PhpParser\Node\Expr;
 
-use PhpParser\Node\ArrayItem;
 use PhpParser\Node\Expr;
 
-class Array_ extends Expr {
+class Array_ extends Expr
+{
     // For use in "kind" attribute
-    public const KIND_LONG = 1;  // array() syntax
+    const KIND_LONG = 1;  // array() syntax
-    public const KIND_SHORT = 2; // [] syntax
+    const KIND_SHORT = 2; // [] syntax
 
-    /** @var ArrayItem[] Items */
+    /** @var (ArrayItem|null)[] Items */
-    public array $items;
+    public $items;
 
     /**
      * Constructs an array node.
      *
-     * @param ArrayItem[] $items Items of the array
+     * @param (ArrayItem|null)[] $items      Items of the array
-     * @param array<string, mixed> $attributes Additional attributes
+     * @param array       $attributes Additional attributes
      */
     public function __construct(array $items = [], array $attributes = []) {
         $this->attributes = $attributes;
         $this->items = $items;
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['items'];
     }
-
+    
-    public function getType(): string {
+    public function getType() : string {
         return 'Expr_Array';
     }
 }

lib/PhpParser/Node/Expr/ArrowFunction.php

@@ -6,60 +6,55 @@ use PhpParser\Node;
 use PhpParser\Node\Expr;
 use PhpParser\Node\FunctionLike;
 
-class ArrowFunction extends Expr implements FunctionLike {
+class ArrowFunction extends Expr implements FunctionLike
-    /** @var bool Whether the closure is static */
+{
-    public bool $static;
+    /** @var bool */
+    public $static;
 
-    /** @var bool Whether to return by reference */
+    /** @var bool */
-    public bool $byRef;
+    public $byRef;
 
     /** @var Node\Param[] */
-    public array $params = [];
+    public $params = [];
 
     /** @var null|Node\Identifier|Node\Name|Node\ComplexType */
-    public ?Node $returnType;
+    public $returnType;
 
-    /** @var Expr Expression body */
+    /** @var Expr */
-    public Expr $expr;
+    public $expr;
     /** @var Node\AttributeGroup[] */
-    public array $attrGroups;
+    public $attrGroups;
 
     /**
-     * @param array{
+     * @param array $subNodes   Array of the following optional subnodes:
-     *     expr: Expr,
+     *                          'static'     => false   : Whether the closure is static
-     *     static?: bool,
+     *                          'byRef'      => false   : Whether to return by reference
-     *     byRef?: bool,
+     *                          'params'     => array() : Parameters
-     *     params?: Node\Param[],
+     *                          'returnType' => null    : Return type
-     *     returnType?: null|Node\Identifier|Node\Name|Node\ComplexType,
+     *                          'expr'       => Expr    : Expression body
-     *     attrGroups?: Node\AttributeGroup[]
+     *                          'attrGroups' => array() : PHP attribute groups
-     * } $subNodes Array of the following subnodes:
+     * @param array $attributes Additional attributes
-     *             'expr'                  : Expression body
-     *             'static'     => false   : Whether the closure is static
-     *             'byRef'      => false   : Whether to return by reference
-     *             'params'     => array() : Parameters
-     *             'returnType' => null    : Return type
-     *             'attrGroups' => array() : PHP attribute groups
-     * @param array<string, mixed> $attributes Additional attributes
      */
-    public function __construct(array $subNodes, array $attributes = []) {
+    public function __construct(array $subNodes = [], array $attributes = []) {
         $this->attributes = $attributes;
         $this->static = $subNodes['static'] ?? false;
         $this->byRef = $subNodes['byRef'] ?? false;
         $this->params = $subNodes['params'] ?? [];
-        $this->returnType = $subNodes['returnType'] ?? null;
+        $returnType = $subNodes['returnType'] ?? null;
+        $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
         $this->expr = $subNodes['expr'];
         $this->attrGroups = $subNodes['attrGroups'] ?? [];
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['attrGroups', 'static', 'byRef', 'params', 'returnType', 'expr'];
     }
 
-    public function returnsByRef(): bool {
+    public function returnsByRef() : bool {
         return $this->byRef;
     }
 
-    public function getParams(): array {
+    public function getParams() : array {
         return $this->params;
     }
 
@@ -67,18 +62,18 @@ class ArrowFunction extends Expr implements FunctionLike {
         return $this->returnType;
     }
 
-    public function getAttrGroups(): array {
+    public function getAttrGroups() : array {
         return $this->attrGroups;
     }
 
     /**
      * @return Node\Stmt\Return_[]
      */
-    public function getStmts(): array {
+    public function getStmts() : array {
         return [new Node\Stmt\Return_($this->expr)];
     }
 
-    public function getType(): string {
+    public function getType() : string {
         return 'Expr_ArrowFunction';
     }
 }

lib/PhpParser/Node/Expr/Assign.php

@@ -4,18 +4,19 @@ namespace PhpParser\Node\Expr;
 
 use PhpParser\Node\Expr;
 
-class Assign extends Expr {
+class Assign extends Expr
+{
     /** @var Expr Variable */
-    public Expr $var;
+    public $var;
     /** @var Expr Expression */
-    public Expr $expr;
+    public $expr;
 
     /**
      * Constructs an assignment node.
      *
-     * @param Expr $var Variable
+     * @param Expr  $var        Variable
-     * @param Expr $expr Expression
+     * @param Expr  $expr       Expression
-     * @param array<string, mixed> $attributes Additional attributes
+     * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
         $this->attributes = $attributes;
@@ -23,11 +24,11 @@ class Assign extends Expr {
         $this->expr = $expr;
     }
 
-    public function getSubNodeNames(): array {
+    public function getSubNodeNames() : array {
         return ['var', 'expr'];
     }
-
+    
-    public function getType(): string {
+    public function getType() : string {
         return 'Expr_Assign';
     }
 }