Update lexer implementations and tests

Each token is now represented by a Token object.

.travis.yml

@@ -1,5 +1,5 @@
 language: php
-dist: trusty
+dist: xenial
 sudo: false
 
 cache:
@@ -10,11 +10,18 @@ php:
   - 7.0
   - 7.1
   - 7.2
+  - 7.3
+  - 7.4snapshot
   - nightly
 
 install:
   - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then composer require satooshi/php-coveralls '~1.0'; fi
-  - composer install --prefer-dist
+  - |
+    if [ $TRAVIS_PHP_VERSION = 'nightly' ]; then
+      composer install --prefer-dist --ignore-platform-reqs;
+    else
+      composer install --prefer-dist;
+    fi
 
 matrix:
   allow_failures:

CHANGELOG.md

@@ -1,8 +1,59 @@
-Version 4.1.1-dev
+Version 4.2.3-dev
 -----------------
 
 Nothing yet.
 
+Version 4.2.2 (2019-05-25)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for arrow functions using a new `Expr\ArrowFunction` node. (#602)
+* [PHP 7.4] Add support for array spreads, using a new `unpack` subnode on `ArrayItem`. (#609)
+* Added support for inserting into empty list nodes in the formatting preserving pretty printer.
+
+### Changed
+
+* `php-parse` will now print messages to stderr, so that stdout only contains the actual result of
+  the operation (such as a JSON dump). (#605)
+
+### Fixed
+
+* Fixed attribute assignment for zero-length nop statements, and a related assertion failure in
+  the formatting-preserving pretty printer. (#589)
+
+Version 4.2.1 (2019-02-16)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for `??=` operator through a new `AssignOp\Coalesce` node. (#575)
+
+Version 4.2.0 (2019-01-12)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for typed properties through a new `type` subnode of `Stmt\Property`.
+  Additionally `Builder\Property` now has a `setType()` method. (#567)
+* Add `kind` attribute to `Cast\Double_`, which allows to distinguish between `(float)`,
+  `(double)` and `(real)`. The form of the cast will be preserved by the pretty printer. (#565)
+
+### Fixed
+
+* Remove assertion when pretty printing anonymous class with a name (#554).
+
+Version 4.1.1 (2018-12-26)
+--------------------------
+
+### Fixed
+
+* Fix "undefined offset" notice when parsing specific malformed code (#551).
+
+### Added
+
+* Support error recovery for missing return type (`function foo() : {}`) (#544).
+
 Version 4.1.0 (2018-10-10)
 --------------------------
 
@@ -17,7 +68,7 @@ Version 4.1.0 (2018-10-10)
      and some cases which we do not expect to occur in practice (such as flexible doc strings being
      nested within each other through abuse of variable-variable interpolation syntax) may not be
      recognized correctly.
-* Added `DONT_TRAVERSER_CURRENT_AND_CHILDREN` to `NodeTraverser` to skip both traversal of child
+* Added `DONT_TRAVERSE_CURRENT_AND_CHILDREN` to `NodeTraverser` to skip both traversal of child
   nodes, and prevent subsequent visitors from visiting the current node.
 
 Version 4.0.4 (2018-09-18)

README.md

@@ -3,7 +3,7 @@ PHP Parser
 
 [![Build Status](https://travis-ci.org/nikic/PHP-Parser.svg?branch=master)](https://travis-ci.org/nikic/PHP-Parser) [![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
 
-This is a PHP 5.2 to PHP 7.2 parser written in PHP. Its purpose is to simplify static code analysis and
+This is a PHP 5.2 to PHP 7.3 parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
 [**Documentation for version 4.x**][doc_master] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 7.3).

bin/php-parse

@@ -45,14 +45,15 @@ $traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);
 foreach ($files as $file) {
     if (strpos($file, '<?php') === 0) {
         $code = $file;
-        echo "====> Code $code\n";
+        fwrite(STDERR, "====> Code $code\n");
     } else {
         if (!file_exists($file)) {
-            die("File $file does not exist.\n");
+            fwrite(STDERR, "File $file does not exist.\n");
+            exit(1);
         }
 
         $code = file_get_contents($file);
-        echo "====> File $file:\n";
+        fwrite(STDERR, "====> File $file:\n");
     }
 
     if ($attributes['with-recovery']) {
@@ -60,7 +61,7 @@ foreach ($files as $file) {
         $stmts = $parser->parse($code, $errorHandler);
         foreach ($errorHandler->getErrors() as $error) {
             $message = formatErrorMessage($error, $code, $attributes['with-column-info']);
-            echo $message . "\n";
+            fwrite(STDERR, $message . "\n");
         }
         if (null === $stmts) {
             continue;
@@ -70,25 +71,26 @@ foreach ($files as $file) {
             $stmts = $parser->parse($code);
         } catch (PhpParser\Error $error) {
             $message = formatErrorMessage($error, $code, $attributes['with-column-info']);
-            die($message . "\n");
+            fwrite(STDERR, $message . "\n");
+            exit(1);
         }
     }
 
     foreach ($operations as $operation) {
         if ('dump' === $operation) {
-            echo "==> Node dump:\n";
+            fwrite(STDERR, "==> Node dump:\n");
             echo $dumper->dump($stmts, $code), "\n";
         } elseif ('pretty-print' === $operation) {
-            echo "==> Pretty print:\n";
+            fwrite(STDERR, "==> Pretty print:\n");
             echo $prettyPrinter->prettyPrintFile($stmts), "\n";
         } elseif ('json-dump' === $operation) {
-            echo "==> JSON dump:\n";
+            fwrite(STDERR, "==> JSON dump:\n");
             echo json_encode($stmts, JSON_PRETTY_PRINT), "\n";
         } elseif ('var-dump' === $operation) {
-            echo "==> var_dump():\n";
+            fwrite(STDERR, "==> var_dump():\n");
             var_dump($stmts);
         } elseif ('resolve-names' === $operation) {
-            echo "==> Resolved names.\n";
+            fwrite(STDERR, "==> Resolved names.\n");
             $stmts = $traverser->traverse($stmts);
         }
     }
@@ -104,9 +106,9 @@ function formatErrorMessage(PhpParser\Error $e, $code, $withColumnInfo) {
 
 function showHelp($error = '') {
     if ($error) {
-        echo $error . "\n\n";
+        fwrite(STDERR, $error . "\n\n");
     }
-    die(<<<OUTPUT
+    fwrite($error ? STDERR : STDOUT, <<<OUTPUT
 Usage: php-parse [operations] file1.php [file2.php ...]
    or: php-parse [operations] "<?php code"
 Turn PHP source code into an abstract syntax tree.
@@ -131,6 +133,7 @@ Example:
 
 OUTPUT
     );
+    exit($error ? 1 : 0);
 }
 
 function parseArgs($args) {

composer.json

@@ -1,8 +1,11 @@
 {
     "name": "nikic/php-parser",
-    "description": "A PHP parser written in PHP",
-    "keywords": ["php", "parser"],
     "type": "library",
+    "description": "A PHP parser written in PHP",
+    "keywords": [
+        "php",
+        "parser"
+    ],
     "license": "BSD-3-Clause",
     "authors": [
         {
@@ -11,20 +14,28 @@
     ],
     "require": {
         "php": ">=7.0",
+        "ext-json": "*",
         "ext-tokenizer": "*"
     },
     "require-dev": {
-        "phpunit/phpunit": "^6.5 || ^7.0"
+        "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0"
+    },
+    "extra": {
+        "branch-alias": {
+            "dev-master": "4.2-dev"
+        }
     },
     "autoload": {
         "psr-4": {
             "PhpParser\\": "lib/PhpParser"
         }
     },
-    "bin": ["bin/php-parse"],
+    "autoload-dev": {
-    "extra": {
+        "psr-4": {
-        "branch-alias": {
+            "PhpParser\\": "test/PhpParser/"
-            "dev-master": "4.1-dev"
         }
-    }
+    },
+    "bin": [
+        "bin/php-parse"
+    ]
 }

doc/0_Introduction.markdown

@@ -1,7 +1,7 @@
 Introduction
 ============
 
-This project is a PHP 5.2 to PHP 7.2 parser **written in PHP itself**.
+This project is a PHP 5.2 to PHP 7.3 parser **written in PHP itself**.
 
 What is this for?
 -----------------
@@ -26,11 +26,11 @@ programmatic PHP code analysis are incidentally PHP developers, not C developers
 What can it parse?
 ------------------
 
-The parser supports parsing PHP 5.2-7.2.
+The parser supports parsing PHP 5.2-7.3.
 
 As the parser is based on the tokens returned by `token_get_all` (which is only able to lex the PHP
 version it runs on), additionally a wrapper for emulating tokens from newer versions is provided.
-This allows to parse PHP 7.2 source code running on PHP 5.5, for example. This emulation is somewhat
+This allows to parse PHP 7.3 source code running on PHP 7.0, for example. This emulation is somewhat
 hacky and not perfect, but it should work well on any sane code.
 
 What output does it produce?
@@ -56,7 +56,7 @@ array(
 ```
 
 This matches the structure of the code: An echo statement, which takes two strings as expressions,
-with the values `Hi` and `World!`.
+with the values `Hi` and `World`.
 
 You can also see that the AST does not contain any whitespace information (but most comments are saved).
 So using it for formatting analysis is not possible.

doc/2_Usage_of_basic_components.markdown

@@ -339,7 +339,8 @@ All four methods can either return the changed node or not return at all (i.e. `
 case the current node is not changed.
 
 The `enterNode()` method can additionally return the value `NodeTraverser::DONT_TRAVERSE_CHILDREN`,
-which instructs the traverser to skip all children of the current node.
+which instructs the traverser to skip all children of the current node. To furthermore prevent subsequent
+visitors from visiting the current node, `NodeTraverser::DONT_TRAVERSE_CURRENT_AND_CHILDREN` can be used instead.
 
 The `leaveNode()` method can additionally return the value `NodeTraverser::REMOVE_NODE`, in which
 case the current node will be removed from the parent array. Furthermore it is possible to return

doc/component/AST_builders.markdown

@@ -48,7 +48,7 @@ $node = $factory->namespace('Name\Space')
             ->makePublic()
             ->makeAbstract() // ->makeFinal()
             ->setReturnType('bool') // ->makeReturnByRef()
-            ->addParam($factory->param('someParam')->setTypeHint('SomeClass'))
+            ->addParam($factory->param('someParam')->setType('SomeClass'))
             ->setDocComment('/**
                               * This method does something.
                               *

doc/component/JSON_representation.markdown

@@ -116,8 +116,8 @@ The JSON representation may be converted back into an AST using the `JsonDecoder
 ```php
 <?php
 
-$nodeDecoder = new PhpParser\NodeDecoder();
+$jsonDecoder = new PhpParser\JsonDecoder();
-$ast = $nodeDecoder->decode($json);
+$ast = $jsonDecoder->decode($json);
 ```
 
 Note that not all ASTs can be represented using JSON. In particular:
@@ -128,4 +128,4 @@ Note that not all ASTs can be represented using JSON. In particular:
 
 If the node tree is not representable in JSON, the initial `json_encode()` call will fail.
 
-From the command line, a JSON dump can be obtained using `vendor/bin/php-parse -j file.php`.
+From the command line, a JSON dump can be obtained using `vendor/bin/php-parse -j file.php`.

doc/component/Walking_the_AST.markdown

@@ -215,7 +215,7 @@ once you found it:
 private $class = null;
 public function enterNode(Node $node) {
     if ($node instanceof Node\Stmt\Class_ &&
-        $node->namespaceName->toString() === 'Foo\Bar\Baz'
+        $node->namespacedName->toString() === 'Foo\Bar\Baz'
     ) {
         $this->class = $node;
         return NodeTraverser::STOP_TRAVERSAL;
@@ -235,7 +235,7 @@ A single traverser can be used with multiple visitors:
 $traverser = new NodeTraverser;
 $traverser->addVisitor($visitorA);
 $traverser->addVisitor($visitorB);
-$stmts = $traverser->traverser($stmts);
+$stmts = $traverser->traverse($stmts);
 ```
 
 It is important to understand that if a traverser is run with multiple visitors, the visitors will
@@ -281,6 +281,8 @@ special enterNode/leaveNode return values:
 
  * If *any* visitor returns `DONT_TRAVERSE_CHILDREN`, the children will be skipped for *all*
    visitors.
+ * If *any* visitor returns `DONT_TRAVERSE_CURRENT_AND_CHILDREN`, the children will be skipped for *all*
+   visitors, and all *subsequent* visitors will not visit the current node.
  * If *any* visitor returns `STOP_TRAVERSAL`, traversal is stopped for *all* visitors.
  * If a visitor returns a replacement node, subsequent visitors will be passed the replacement node,
    not the original one.
@@ -305,7 +307,7 @@ $nodeFinder = new NodeFinder;
 $classes = $nodeFinder->findInstanceOf($stmts, Node\Stmt\Class_::class);
 
 // Find all classes that extend another class
-$extendingClasses = $nodeFinder->findInstanceOf($stmts, function(Node $node) {
+$extendingClasses = $nodeFinder->find($stmts, function(Node $node) {
     return $node instanceof Node\Stmt\Class_
         && $node->extends !== null;
 });
@@ -332,4 +334,4 @@ reverse direction: When working on a node, you might want to check if the parent
 certain property.
 
 PHP-Parser does not add parent (or sibling) references to nodes by itself, but you can easily
-emulate this with a visitor. See the [FAQ](FAQ.markdown) for more information.
+emulate this with a visitor. See the [FAQ](FAQ.markdown) for more information.

grammar/php5.y

@@ -16,7 +16,7 @@ top_statement_list_ex:
 
 top_statement_list:
       top_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -27,7 +27,7 @@ reserved_non_modifiers:
     | T_FINALLY | T_THROW | T_USE | T_INSTEADOF | T_GLOBAL | T_VAR | T_UNSET | T_ISSET | T_EMPTY | T_CONTINUE | T_GOTO
     | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT
     | T_BREAK | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE | T_CLASS
-    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER
+    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER | T_FN
 ;
 
 semi_reserved:
@@ -160,7 +160,7 @@ inner_statement_list_ex:
 
 inner_statement_list:
       inner_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -461,7 +461,7 @@ class_statement_list_ex:
 
 class_statement_list:
       class_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -579,6 +579,7 @@ expr:
     | variable T_SL_EQUAL expr                              { $$ = Expr\AssignOp\ShiftLeft [$1, $3]; }
     | variable T_SR_EQUAL expr                              { $$ = Expr\AssignOp\ShiftRight[$1, $3]; }
     | variable T_POW_EQUAL expr                             { $$ = Expr\AssignOp\Pow       [$1, $3]; }
+    | variable T_COALESCE_EQUAL expr                        { $$ = Expr\AssignOp\Coalesce  [$1, $3]; }
     | variable T_INC                                        { $$ = Expr\PostInc[$1]; }
     | T_INC variable                                        { $$ = Expr\PreInc [$2]; }
     | variable T_DEC                                        { $$ = Expr\PostDec[$1]; }
@@ -628,7 +629,10 @@ expr:
     | T_REQUIRE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE]; }
     | T_REQUIRE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE_ONCE]; }
     | T_INT_CAST expr                                       { $$ = Expr\Cast\Int_    [$2]; }
-    | T_DOUBLE_CAST expr                                    { $$ = Expr\Cast\Double  [$2]; }
+    | T_DOUBLE_CAST expr
+          { $attrs = attributes();
+            $attrs['kind'] = $this->getFloatCastKind($1);
+            $$ = new Expr\Cast\Double($2, $attrs); }
     | T_STRING_CAST expr                                    { $$ = Expr\Cast\String_ [$2]; }
     | T_ARRAY_CAST expr                                     { $$ = Expr\Cast\Array_  [$2]; }
     | T_OBJECT_CAST expr                                    { $$ = Expr\Cast\Object_ [$2]; }
@@ -984,6 +988,7 @@ array_pair:
     | expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
     | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
     | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
+    | T_ELLIPSIS expr                                       { $$ = Expr\ArrayItem[$2, null, false, attributes(), true]; }
 ;
 
 encaps_list:

grammar/php7.y

@@ -16,7 +16,7 @@ top_statement_list_ex:
 
 top_statement_list:
       top_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -27,7 +27,7 @@ reserved_non_modifiers:
     | T_FINALLY | T_THROW | T_USE | T_INSTEADOF | T_GLOBAL | T_VAR | T_UNSET | T_ISSET | T_EMPTY | T_CONTINUE | T_GOTO
     | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT
     | T_BREAK | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE | T_CLASS
-    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER
+    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER | T_FN
 ;
 
 semi_reserved:
@@ -196,7 +196,7 @@ inner_statement_list_ex:
 
 inner_statement_list:
       inner_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -450,11 +450,11 @@ non_empty_parameter_list:
 ;
 
 parameter:
-      optional_param_type optional_ref optional_ellipsis plain_variable
+      optional_type optional_ref optional_ellipsis plain_variable
           { $$ = Node\Param[$4, null, $1, $2, $3]; $this->checkParam($$); }
-    | optional_param_type optional_ref optional_ellipsis plain_variable '=' expr
+    | optional_type optional_ref optional_ellipsis plain_variable '=' expr
           { $$ = Node\Param[$4, $6, $1, $2, $3]; $this->checkParam($$); }
-    | optional_param_type optional_ref optional_ellipsis error
+    | optional_type optional_ref optional_ellipsis error
           { $$ = Node\Param[Expr\Error[], null, $1, $2, $3]; }
 ;
 
@@ -469,7 +469,7 @@ type:
     | T_CALLABLE                                            { $$ = Node\Identifier['callable']; }
 ;
 
-optional_param_type:
+optional_type:
       /* empty */                                           { $$ = null; }
     | type_expr                                             { $$ = $1; }
 ;
@@ -477,6 +477,7 @@ optional_param_type:
 optional_return_type:
       /* empty */                                           { $$ = null; }
     | ':' type_expr                                         { $$ = $2; }
+    | ':' error                                             { $$ = null; }
 ;
 
 argument_list:
@@ -529,13 +530,14 @@ class_statement_list_ex:
 
 class_statement_list:
       class_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
 class_statement:
-      variable_modifiers property_declaration_list ';'
+      variable_modifiers optional_type property_declaration_list ';'
-          { $$ = Stmt\Property[$1, $2]; $this->checkProperty($$, #1); }
+          { $attrs = attributes();
+            $$ = new Stmt\Property($1, $3, $attrs, $2); $this->checkProperty($$, #1); }
     | method_modifiers T_CONST class_const_list ';'
           { $$ = Stmt\ClassConst[$3, $1]; $this->checkClassConst($$, #1); }
     | method_modifiers T_FUNCTION optional_ref identifier_ex '(' parameter_list ')' optional_return_type method_body
@@ -658,6 +660,7 @@ expr:
     | variable T_SL_EQUAL expr                              { $$ = Expr\AssignOp\ShiftLeft [$1, $3]; }
     | variable T_SR_EQUAL expr                              { $$ = Expr\AssignOp\ShiftRight[$1, $3]; }
     | variable T_POW_EQUAL expr                             { $$ = Expr\AssignOp\Pow       [$1, $3]; }
+    | variable T_COALESCE_EQUAL expr                        { $$ = Expr\AssignOp\Coalesce  [$1, $3]; }
     | variable T_INC                                        { $$ = Expr\PostInc[$1]; }
     | T_INC variable                                        { $$ = Expr\PreInc [$2]; }
     | variable T_DEC                                        { $$ = Expr\PostDec[$1]; }
@@ -705,7 +708,10 @@ expr:
     | T_REQUIRE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE]; }
     | T_REQUIRE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE_ONCE]; }
     | T_INT_CAST expr                                       { $$ = Expr\Cast\Int_    [$2]; }
-    | T_DOUBLE_CAST expr                                    { $$ = Expr\Cast\Double  [$2]; }
+    | T_DOUBLE_CAST expr
+          { $attrs = attributes();
+            $attrs['kind'] = $this->getFloatCastKind($1);
+            $$ = new Expr\Cast\Double($2, $attrs); }
     | T_STRING_CAST expr                                    { $$ = Expr\Cast\String_ [$2]; }
     | T_ARRAY_CAST expr                                     { $$ = Expr\Cast\Array_  [$2]; }
     | T_OBJECT_CAST expr                                    { $$ = Expr\Cast\Object_ [$2]; }
@@ -723,6 +729,12 @@ expr:
     | T_YIELD expr                                          { $$ = Expr\Yield_[$2, null]; }
     | T_YIELD expr T_DOUBLE_ARROW expr                      { $$ = Expr\Yield_[$4, $2]; }
     | T_YIELD_FROM expr                                     { $$ = Expr\YieldFrom[$2]; }
+
+    | T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr
+          { $$ = Expr\ArrowFunction[['static' => false, 'byRef' => $2, 'params' => $4, 'returnType' => $6, 'expr' => $8]]; }
+    | T_STATIC T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr
+          { $$ = Expr\ArrowFunction[['static' => true, 'byRef' => $3, 'params' => $5, 'returnType' => $7, 'expr' => $9]]; }
+
     | T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type
       block_or_error
           { $$ = Expr\Closure[['static' => false, 'byRef' => $2, 'params' => $4, 'uses' => $6, 'returnType' => $7, 'stmts' => $8]]; }
@@ -958,6 +970,7 @@ array_pair_list:
 comma_or_error:
       ','
     | error
+          { /* do nothing -- prevent default action of $$=$1. See #551. */ }
 ;
 
 inner_array_pair_list:
@@ -970,6 +983,7 @@ array_pair:
     | expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
     | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
     | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
+    | T_ELLIPSIS expr                                       { $$ = Expr\ArrayItem[$2, null, false, attributes(), true]; }
     | /* empty */                                           { $$ = null; }
 ;
 

grammar/rebuildParsers.php

@@ -13,9 +13,12 @@ $tmpResultFile  = __DIR__ . '/tmp_parser.php';
 $resultDir = __DIR__ . '/../lib/PhpParser/Parser';
 $tokensResultsFile = $resultDir . '/Tokens.php';
 
-// check for kmyacc.exe binary in this directory, otherwise fall back to global name
+// check for kmyacc binary in this directory, otherwise fall back to global name
-$kmyacc = __DIR__ . '/kmyacc.exe';
+if (file_exists(__DIR__ . '/kmyacc.exe')) {
-if (!file_exists($kmyacc)) {
+    $kmyacc = __DIR__ . '/kmyacc.exe';
+} else if (file_exists(__DIR__ . '/kmyacc')) {
+    $kmyacc = __DIR__ . '/kmyacc';
+} else {
     $kmyacc = 'kmyacc';
 }
 
@@ -175,6 +178,15 @@ function resolveMacros($code) {
                 . ' else { ' . $args[0] . ' = null; }';
             }
 
+            if ('makeZeroLengthNop' == $name) {
+                assertArgs(2, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                    . ' if (isset($startAttributes[\'comments\']))'
+                    . ' { ' . $args[0] . ' = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); }'
+                    . ' else { ' . $args[0] . ' = null; }';
+            }
+
             if ('strKind' == $name) {
                 assertArgs(1, $args, $name);
 

grammar/tokens.y

@@ -10,7 +10,7 @@
 %right T_YIELD
 %right T_DOUBLE_ARROW
 %right T_YIELD_FROM
-%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL
+%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL T_COALESCE_EQUAL
 %left '?' ':'
 %right T_COALESCE
 %left T_BOOLEAN_OR
@@ -64,6 +64,7 @@
 %token T_CONTINUE
 %token T_GOTO
 %token T_FUNCTION
+%token T_FN
 %token T_CONST
 %token T_RETURN
 %token T_TRY

lib/PhpParser/Builder/Property.php

@@ -4,6 +4,9 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Name;
+use PhpParser\Node\NullableType;
 use PhpParser\Node\Stmt;
 
 class Property implements PhpParser\Builder
@@ -14,6 +17,9 @@ class Property implements PhpParser\Builder
     protected $default = null;
     protected $attributes = [];
 
+    /** @var null|Identifier|Name|NullableType */
+    protected $type;
+
     /**
      * Creates a property builder.
      *
@@ -95,6 +101,19 @@ class Property implements PhpParser\Builder
         return $this;
     }
 
+    /**
+     * Sets the property type for PHP 7.4+.
+     *
+     * @param string|Name|NullableType|Identifier $type
+     *
+     * @return $this
+     */
+    public function setType($type) {
+        $this->type = BuilderHelpers::normalizeType($type);
+
+        return $this;
+    }
+
     /**
      * Returns the built class node.
      *
@@ -106,7 +125,8 @@ class Property implements PhpParser\Builder
             [
                 new Stmt\PropertyProperty($this->name, $this->default)
             ],
-            $this->attributes
+            $this->attributes,
+            $this->type
         );
     }
 }

lib/PhpParser/BuilderFactory.php

@@ -8,7 +8,6 @@ use PhpParser\Node\Expr\BinaryOp\Concat;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
 use PhpParser\Node\Scalar\String_;
-use PhpParser\Node\Stmt;
 use PhpParser\Node\Stmt\Use_;
 
 class BuilderFactory
@@ -77,7 +76,7 @@ class BuilderFactory
      * @return Builder\TraitUseAdaptation The create trait use adaptation builder
      */
     public function traitUseAdaptation($trait, $method = null) : Builder\TraitUseAdaptation {
-        if (is_null($method)) {
+        if ($method === null) {
             $method = $trait;
             $trait = null;
         }

lib/PhpParser/FileContext.php

@@ -0,0 +1,8 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser;
+
+class FileContext {
+    /** @var Token[] */
+    public $tokens;
+}

lib/PhpParser/Internal/PrintableNewAnonClassNode.php

@@ -39,7 +39,8 @@ class PrintableNewAnonClassNode extends Expr
     public static function fromNewNode(Expr\New_ $newNode) {
         $class = $newNode->class;
         assert($class instanceof Node\Stmt\Class_);
-        assert($class->name === null);
+        // We don't assert that $class->name is null here, to allow consumers to assign unique names
+        // to anonymous classes for their own purposes. We simplify ignore the name here.
         return new self(
             $newNode->args, $class->extends, $class->implements,
             $class->stmts, $newNode->getAttributes()

lib/PhpParser/Internal/TokenStream.php

@@ -2,6 +2,8 @@
 
 namespace PhpParser\Internal;
 
+use PhpParser\Token;
+
 /**
  * Provides operations on token streams, for use by pretty printer.
  *
@@ -9,7 +11,7 @@ namespace PhpParser\Internal;
  */
 class TokenStream
 {
-    /** @var array Tokens (in token_get_all format) */
+    /** @var Token[] */
     private $tokens;
     /** @var int[] Map from position to indentation */
     private $indentMap;
@@ -17,7 +19,7 @@ class TokenStream
     /**
      * Create token stream instance.
      *
-     * @param array $tokens Tokens in token_get_all() format
+     * @param Token[] $tokens Tokens
      */
     public function __construct(array $tokens) {
         $this->tokens = $tokens;
@@ -33,8 +35,8 @@ class TokenStream
      * @return bool
      */
     public function haveParens(int $startPos, int $endPos) : bool {
-        return $this->haveTokenImmediativelyBefore($startPos, '(')
+        return $this->haveTokenImmediativelyBefore($startPos, \ord('('))
-            && $this->haveTokenImmediatelyAfter($endPos, ')');
+            && $this->haveTokenImmediatelyAfter($endPos, \ord(')'));
     }
 
     /**
@@ -46,8 +48,8 @@ class TokenStream
      * @return bool
      */
     public function haveBraces(int $startPos, int $endPos) : bool {
-        return $this->haveTokenImmediativelyBefore($startPos, '{')
+        return $this->haveTokenImmediativelyBefore($startPos, \ord('{'))
-            && $this->haveTokenImmediatelyAfter($endPos, '}');
+            && $this->haveTokenImmediatelyAfter($endPos, \ord('}'));
     }
 
     /**
@@ -64,7 +66,7 @@ class TokenStream
         $tokens = $this->tokens;
         $pos--;
         for (; $pos >= 0; $pos--) {
-            $tokenType = $tokens[$pos][0];
+            $tokenType = $tokens[$pos]->id;
             if ($tokenType === $expectedTokenType) {
                 return true;
             }
@@ -90,7 +92,7 @@ class TokenStream
         $tokens = $this->tokens;
         $pos++;
         for (; $pos < \count($tokens); $pos++) {
-            $tokenType = $tokens[$pos][0];
+            $tokenType = $tokens[$pos]->id;
             if ($tokenType === $expectedTokenType) {
                 return true;
             }
@@ -110,7 +112,7 @@ class TokenStream
             return $pos;
         }
 
-        if ($tokens[$pos][0] !== $skipTokenType) {
+        if ($tokens[$pos]->id !== $skipTokenType) {
             // Shouldn't happen. The skip token MUST be there
             throw new \Exception('Encountered unexpected token');
         }
@@ -127,7 +129,7 @@ class TokenStream
             return $pos;
         }
 
-        if ($tokens[$pos][0] !== $skipTokenType) {
+        if ($tokens[$pos]->id !== $skipTokenType) {
             // Shouldn't happen. The skip token MUST be there
             throw new \Exception('Encountered unexpected token');
         }
@@ -145,7 +147,7 @@ class TokenStream
     public function skipLeftWhitespace(int $pos) {
         $tokens = $this->tokens;
         for (; $pos >= 0; $pos--) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
             if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
@@ -162,7 +164,7 @@ class TokenStream
     public function skipRightWhitespace(int $pos) {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
             if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                 break;
             }
@@ -173,7 +175,7 @@ class TokenStream
     public function findRight($pos, $findTokenType) {
         $tokens = $this->tokens;
         for ($count = \count($tokens); $pos < $count; $pos++) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
             if ($type === $findTokenType) {
                 return $pos;
             }
@@ -206,23 +208,19 @@ class TokenStream
         $result = '';
         for ($pos = $from; $pos < $to; $pos++) {
             $token = $tokens[$pos];
-            if (\is_array($token)) {
+            $type = $token->id;
-                $type = $token[0];
+            $content = $token->value;
-                $content = $token[1];
+            if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
-                if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
+                $result .= $content;
-                    $result .= $content;
-                } else {
-                    // TODO Handle non-space indentation
-                    if ($indent < 0) {
-                        $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
-                    } elseif ($indent > 0) {
-                        $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
-                    } else {
-                        $result .= $content;
-                    }
-                }
             } else {
-                $result .= $token;
+                // TODO Handle non-space indentation
+                if ($indent < 0) {
+                    $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
+                } elseif ($indent > 0) {
+                    $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
+                } else {
+                    $result .= $content;
+                }
             }
         }
         return $result;
@@ -239,8 +237,8 @@ class TokenStream
         foreach ($this->tokens as $token) {
             $indentMap[] = $indent;
 
-            if ($token[0] === \T_WHITESPACE) {
+            if ($token->id === \T_WHITESPACE) {
-                $content = $token[1];
+                $content = $token->value;
                 $newlinePos = \strrpos($content, "\n");
                 if (false !== $newlinePos) {
                     $indent = \strlen($content) - $newlinePos - 1;

lib/PhpParser/Lexer.php

@@ -6,88 +6,132 @@ use PhpParser\Parser\Tokens;
 
 class Lexer
 {
-    protected $code;
+    /** @var array Map from PHP tokens to PhpParser tokens. */
-    protected $tokens;
-    protected $pos;
-    protected $line;
-    protected $filePos;
-    protected $prevCloseTagHasNewline;
-
     protected $tokenMap;
-    protected $dropTokens;
-
-    protected $usedAttributes;
 
     /**
      * Creates a Lexer.
      *
-     * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
+     * @param array $options Options array. Currently unused.
-     *                       which is an array of attributes to add to the AST nodes. Possible
-     *                       attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
-     *                       'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
-     *                       first three. For more info see getNextToken() docs.
      */
     public function __construct(array $options = []) {
-        // map from internal tokens to PhpParser tokens
         $this->tokenMap = $this->createTokenMap();
-
-        // map of tokens to drop while lexing (the map is only used for isset lookup,
-        // that's why the value is simply set to 1; the value is never actually used.)
-        $this->dropTokens = array_fill_keys(
-            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT], 1
-        );
-
-        // the usedAttributes member is a map of the used attribute names to a dummy
-        // value (here "true")
-        $options += [
-            'usedAttributes' => ['comments', 'startLine', 'endLine'],
-        ];
-        $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
     }
 
     /**
-     * Initializes the lexer for lexing the provided source code.
+     * Get tokens IDs that should be ignored by the parser.
+     *
+     * @return array
+     */
+    public function getIgnorableTokens(): array {
+        return [
+            Tokens::T_WHITESPACE,
+            Tokens::T_COMMENT,
+            Tokens::T_DOC_COMMENT,
+            Tokens::T_OPEN_TAG,
+            Tokens::T_BAD_CHARACTER,
+        ];
+    }
+
+    /**
+     * Get map for token canonicalization.
+     *
+     * @return array
+     */
+    public function getCanonicalizationMap(): array {
+        return [
+            Tokens::T_OPEN_TAG_WITH_ECHO => Tokens::T_ECHO,
+            Tokens::T_CLOSE_TAG => \ord(';'),
+        ];
+    }
+
+    /**
+     * Tokenizes the given PHP code into an array of Tokens.
      *
      * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
      * the getErrors() method.
      *
-     * @param string $code The source code to lex
+     * @param string $code The source code to tokenize
      * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
      *                                        ErrorHandler\Throwing
+     *
+     * @return Token[] Sequence of tokens
      */
-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+    public function tokenize(string $code, ErrorHandler $errorHandler = null) {
         if (null === $errorHandler) {
             $errorHandler = new ErrorHandler\Throwing();
         }
 
-        $this->code = $code; // keep the code around for __halt_compiler() handling
-        $this->pos  = -1;
-        $this->line =  1;
-        $this->filePos = 0;
-
-        // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
-        // This ensures proper composability, because having a newline is the "safe" assumption.
-        $this->prevCloseTagHasNewline = true;
-
         $scream = ini_set('xdebug.scream', '0');
-
         error_clear_last();
-        $this->tokens = @token_get_all($code);
+        $rawTokens = @token_get_all($code);
-        $this->handleErrors($errorHandler);
+        $checkForMissingTokens = null !== error_get_last();
-
         if (false !== $scream) {
             ini_set('xdebug.scream', $scream);
         }
-    }
 
-    private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
+        $tokens = [];
-        for ($i = $start; $i < $end; $i++) {
+        $filePos = 0;
-            $chr = $this->code[$i];
+        $line = 1;
-            if ($chr === 'b' || $chr === 'B') {
+        foreach ($rawTokens as $rawToken) {
-                // HHVM does not treat b" tokens correctly, so ignore these
+            if (\is_array($rawToken)) {
-                continue;
+                $token = new Token($this->tokenMap[$rawToken[0]], $rawToken[1], $line, $filePos);
+            } elseif (\strlen($rawToken) == 2) {
+                // Bug in token_get_all() when lexing b".
+                $token = new Token(\ord('"'), $rawToken, $line, $filePos);
+            } else {
+                $token = new Token(\ord($rawToken), $rawToken, $line, $filePos);
             }
 
+            $value = $token->value;
+            $tokenLen = \strlen($value);
+            if ($checkForMissingTokens && substr($code, $filePos, $tokenLen) !== $value) {
+                // Something is missing, must be an invalid character
+                $nextFilePos = strpos($code, $value, $filePos);
+                $badCharTokens = $this->handleInvalidCharacterRange(
+                    $code, $filePos, $nextFilePos, $line, $errorHandler);
+                $tokens = array_merge($tokens, $badCharTokens);
+                $filePos = (int) $nextFilePos;
+            }
+
+            $tokens[] = $token;
+            $filePos += $tokenLen;
+            $line += substr_count($value, "\n");
+        }
+
+        if ($filePos !== \strlen($code)) {
+            // Invalid characters at the end of the input
+            $badCharTokens = $this->handleInvalidCharacterRange(
+                $code, $filePos, \strlen($code), $line, $errorHandler);
+            $tokens = array_merge($tokens, $badCharTokens);
+        }
+
+        if (\count($tokens) > 0) {
+            // Check for unterminated comment
+            $lastToken = $tokens[\count($tokens) - 1];
+            if ($this->isUnterminatedComment($lastToken)) {
+                $errorHandler->handleError(new Error('Unterminated comment', [
+                    'startLine' => $line - substr_count($lastToken->value, "\n"),
+                    'endLine' => $line,
+                    'startFilePos' => $filePos - \strlen($lastToken->value),
+                    'endFilePos' => $filePos,
+                ]));
+            }
+        }
+
+        // Add an EOF sentinel token
+        // TODO: Should the value be an empty string instead?
+        $tokens[] = new Token(0, "\0", $line, \strlen($code));
+
+        return $tokens;
+    }
+
+    private function handleInvalidCharacterRange(
+        string $code, int $start, int $end, int $line, ErrorHandler $errorHandler
+    ) {
+        $tokens = [];
+        for ($i = $start; $i < $end; $i++) {
+            $chr = $code[$i];
             if ($chr === "\0") {
                 // PHP cuts error message after null byte, so need special case
                 $errorMsg = 'Unexpected null byte';
@@ -97,6 +141,7 @@ class Lexer
                 );
             }
 
+            $tokens[] = new Token(Tokens::T_BAD_CHARACTER, $chr, $line, $i);
             $errorHandler->handleError(new Error($errorMsg, [
                 'startLine' => $line,
                 'endLine' => $line,
@@ -104,273 +149,27 @@ class Lexer
                 'endFilePos' => $i,
             ]));
         }
+        return $tokens;
     }
 
-    /**
+    private function isUnterminatedComment(Token $token): bool {
-     * Check whether comment token is unterminated.
+        return ($token->id === \T_COMMENT || $token->id === \T_DOC_COMMENT)
-     *
+            && substr($token->value, 0, 2) === '/*'
-     * @return bool
+            && substr($token->value, -2) !== '*/';
-     */
-    private function isUnterminatedComment($token) : bool {
-        return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
-            && substr($token[1], 0, 2) === '/*'
-            && substr($token[1], -2) !== '*/';
     }
 
-    /**
+    private function createTokenMap(): array {
-     * Check whether an error *may* have occurred during tokenization.
-     *
-     * @return bool
-     */
-    private function errorMayHaveOccurred() : bool {
-        if (defined('HHVM_VERSION')) {
-            // In HHVM token_get_all() does not throw warnings, so we need to conservatively
-            // assume that an error occurred
-            return true;
-        }
-
-        return null !== error_get_last();
-    }
-
-    protected function handleErrors(ErrorHandler $errorHandler) {
-        if (!$this->errorMayHaveOccurred()) {
-            return;
-        }
-
-        // PHP's error handling for token_get_all() is rather bad, so if we want detailed
-        // error information we need to compute it ourselves. Invalid character errors are
-        // detected by finding "gaps" in the token array. Unterminated comments are detected
-        // by checking if a trailing comment has a "*/" at the end.
-
-        $filePos = 0;
-        $line = 1;
-        foreach ($this->tokens as $token) {
-            $tokenValue = \is_string($token) ? $token : $token[1];
-            $tokenLen = \strlen($tokenValue);
-
-            if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
-                // Something is missing, must be an invalid character
-                $nextFilePos = strpos($this->code, $tokenValue, $filePos);
-                $this->handleInvalidCharacterRange(
-                    $filePos, $nextFilePos, $line, $errorHandler);
-                $filePos = (int) $nextFilePos;
-            }
-
-            $filePos += $tokenLen;
-            $line += substr_count($tokenValue, "\n");
-        }
-
-        if ($filePos !== \strlen($this->code)) {
-            if (substr($this->code, $filePos, 2) === '/*') {
-                // Unlike PHP, HHVM will drop unterminated comments entirely
-                $comment = substr($this->code, $filePos);
-                $errorHandler->handleError(new Error('Unterminated comment', [
-                    'startLine' => $line,
-                    'endLine' => $line + substr_count($comment, "\n"),
-                    'startFilePos' => $filePos,
-                    'endFilePos' => $filePos + \strlen($comment),
-                ]));
-
-                // Emulate the PHP behavior
-                $isDocComment = isset($comment[3]) && $comment[3] === '*';
-                $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
-            } else {
-                // Invalid characters at the end of the input
-                $this->handleInvalidCharacterRange(
-                    $filePos, \strlen($this->code), $line, $errorHandler);
-            }
-            return;
-        }
-
-        if (count($this->tokens) > 0) {
-            // Check for unterminated comment
-            $lastToken = $this->tokens[count($this->tokens) - 1];
-            if ($this->isUnterminatedComment($lastToken)) {
-                $errorHandler->handleError(new Error('Unterminated comment', [
-                    'startLine' => $line - substr_count($lastToken[1], "\n"),
-                    'endLine' => $line,
-                    'startFilePos' => $filePos - \strlen($lastToken[1]),
-                    'endFilePos' => $filePos,
-                ]));
-            }
-        }
-    }
-
-    /**
-     * Fetches the next token.
-     *
-     * The available attributes are determined by the 'usedAttributes' option, which can
-     * be specified in the constructor. The following attributes are supported:
-     *
-     *  * 'comments'      => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
-     *                       representing all comments that occurred between the previous
-     *                       non-discarded token and the current one.
-     *  * 'startLine'     => Line in which the node starts.
-     *  * 'endLine'       => Line in which the node ends.
-     *  * 'startTokenPos' => Offset into the token array of the first token in the node.
-     *  * 'endTokenPos'   => Offset into the token array of the last token in the node.
-     *  * 'startFilePos'  => Offset into the code string of the first character that is part of the node.
-     *  * 'endFilePos'    => Offset into the code string of the last character that is part of the node.
-     *
-     * @param mixed $value           Variable to store token content in
-     * @param mixed $startAttributes Variable to store start attributes in
-     * @param mixed $endAttributes   Variable to store end attributes in
-     *
-     * @return int Token id
-     */
-    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
-        $startAttributes = [];
-        $endAttributes   = [];
-
-        while (1) {
-            if (isset($this->tokens[++$this->pos])) {
-                $token = $this->tokens[$this->pos];
-            } else {
-                // EOF token with ID 0
-                $token = "\0";
-            }
-
-            if (isset($this->usedAttributes['startLine'])) {
-                $startAttributes['startLine'] = $this->line;
-            }
-            if (isset($this->usedAttributes['startTokenPos'])) {
-                $startAttributes['startTokenPos'] = $this->pos;
-            }
-            if (isset($this->usedAttributes['startFilePos'])) {
-                $startAttributes['startFilePos'] = $this->filePos;
-            }
-
-            if (\is_string($token)) {
-                $value = $token;
-                if (isset($token[1])) {
-                    // bug in token_get_all
-                    $this->filePos += 2;
-                    $id = ord('"');
-                } else {
-                    $this->filePos += 1;
-                    $id = ord($token);
-                }
-            } elseif (!isset($this->dropTokens[$token[0]])) {
-                $value = $token[1];
-                $id = $this->tokenMap[$token[0]];
-                if (\T_CLOSE_TAG === $token[0]) {
-                    $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
-                } elseif (\T_INLINE_HTML === $token[0]) {
-                    $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
-                }
-
-                $this->line += substr_count($value, "\n");
-                $this->filePos += \strlen($value);
-            } else {
-                if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
-                    if (isset($this->usedAttributes['comments'])) {
-                        $comment = \T_DOC_COMMENT === $token[0]
-                            ? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
-                            : new Comment($token[1], $this->line, $this->filePos, $this->pos);
-                        $startAttributes['comments'][] = $comment;
-                    }
-                }
-
-                $this->line += substr_count($token[1], "\n");
-                $this->filePos += \strlen($token[1]);
-                continue;
-            }
-
-            if (isset($this->usedAttributes['endLine'])) {
-                $endAttributes['endLine'] = $this->line;
-            }
-            if (isset($this->usedAttributes['endTokenPos'])) {
-                $endAttributes['endTokenPos'] = $this->pos;
-            }
-            if (isset($this->usedAttributes['endFilePos'])) {
-                $endAttributes['endFilePos'] = $this->filePos - 1;
-            }
-
-            return $id;
-        }
-
-        throw new \RuntimeException('Reached end of lexer loop');
-    }
-
-    /**
-     * Returns the token array for current code.
-     *
-     * The token array is in the same format as provided by the
-     * token_get_all() function and does not discard tokens (i.e.
-     * whitespace and comments are included). The token position
-     * attributes are against this token array.
-     *
-     * @return array Array of tokens in token_get_all() format
-     */
-    public function getTokens() : array {
-        return $this->tokens;
-    }
-
-    /**
-     * Handles __halt_compiler() by returning the text after it.
-     *
-     * @return string Remaining text
-     */
-    public function handleHaltCompiler() : string {
-        // text after T_HALT_COMPILER, still including ();
-        $textAfter = substr($this->code, $this->filePos);
-
-        // ensure that it is followed by ();
-        // this simplifies the situation, by not allowing any comments
-        // in between of the tokens.
-        if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
-            throw new Error('__HALT_COMPILER must be followed by "();"');
-        }
-
-        // prevent the lexer from returning any further tokens
-        $this->pos = count($this->tokens);
-
-        // return with (); removed
-        return substr($textAfter, strlen($matches[0]));
-    }
-
-    /**
-     * Creates the token map.
-     *
-     * The token map maps the PHP internal token identifiers
-     * to the identifiers used by the Parser. Additionally it
-     * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
-     *
-     * @return array The token map
-     */
-    protected function createTokenMap() : array {
         $tokenMap = [];
 
-        // 256 is the minimum possible token number, as everything below
-        // it is an ASCII value
         for ($i = 256; $i < 1000; ++$i) {
-            if (\T_DOUBLE_COLON === $i) {
+            $name = token_name($i);
-                // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
+            if ('UNKNOWN' === $name) {
-                $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
+                continue;
-            } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
+            }
-                // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
+            $constName = Tokens::class . '::' . $name;
-                $tokenMap[$i] = Tokens::T_ECHO;
+            if (defined($constName)) {
-            } elseif(\T_CLOSE_TAG === $i) {
+                $tokenMap[$i] = constant($constName);
-                // T_CLOSE_TAG is equivalent to ';'
-                $tokenMap[$i] = ord(';');
-            } elseif ('UNKNOWN' !== $name = token_name($i)) {
-                if ('T_HASHBANG' === $name) {
-                    // HHVM uses a special token for #! hashbang lines
-                    $tokenMap[$i] = Tokens::T_INLINE_HTML;
-                } elseif (defined($name = Tokens::class . '::' . $name)) {
-                    // Other tokens can be mapped directly
-                    $tokenMap[$i] = constant($name);
-                }
             }
-        }
-
-        // HHVM uses a special token for numbers that overflow to double
-        if (defined('T_ONUMBER')) {
-            $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
-        }
-        // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
-        if (defined('T_COMPILER_HALT_OFFSET')) {
-            $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
         }
 
         return $tokenMap;

lib/PhpParser/Lexer/Emulative.php

@@ -4,28 +4,56 @@ namespace PhpParser\Lexer;
 
 use PhpParser\Error;
 use PhpParser\ErrorHandler;
+use PhpParser\Lexer;
+use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
+use PhpParser\Lexer\TokenEmulator\TokenEmulatorInterface;
+use PhpParser\Parser\Tokens;
 
-class Emulative extends \PhpParser\Lexer
+class Emulative extends Lexer
 {
     const PHP_7_3 = '7.3.0dev';
+    const PHP_7_4 = '7.4.0dev';
+
+    const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
+/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
+(?:.*\r?\n)*?
+(?<indentation>\h*)\2(?![a-zA-Z_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
+REGEX;
+
+    /** @var mixed[] Patches used to reverse changes introduced in the code */
+    private $patches = [];
+
+    /** @var TokenEmulatorInterface[] */
+    private $tokenEmulators = [];
 
     /**
-     * @var array Patches used to reverse changes introduced in the code
+     * @param mixed[] $options
      */
-    private $patches;
+    public function __construct(array $options = [])
+    {
+        parent::__construct($options);
 
-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+        $this->tokenEmulators[] = new FnTokenEmulator();
+        $this->tokenEmulators[] = new CoaleseEqualTokenEmulator();
+        $this->tokenEmulators[] = new NumericLiteralSeparatorEmulator();
+    }
+
+    public function tokenize(string $code, ErrorHandler $errorHandler = null) {
         $this->patches = [];
-        $preparedCode = $this->prepareCode($code);
+
-        if (null === $preparedCode) {
+        if ($this->isEmulationNeeded($code) === false) {
             // Nothing to emulate, yay
-            parent::startLexing($code, $errorHandler);
+            return parent::tokenize($code, $errorHandler);
-            return;
         }
 
         $collector = new ErrorHandler\Collecting();
-        parent::startLexing($preparedCode, $collector);
+
-        $this->fixupTokens();
+        // 1. emulation of heredoc and nowdoc new syntax
+        $preparedCode = $this->processHeredocNowdoc($code);
+        $tokens = parent::tokenize($preparedCode, $collector);
+        $tokens = $this->fixupTokens($tokens);
 
         $errors = $collector->getErrors();
         if (!empty($errors)) {
@@ -34,32 +62,36 @@ class Emulative extends \PhpParser\Lexer
                 $errorHandler->handleError($error);
             }
         }
+
+        // add token emulation
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                $tokens = $emulativeToken->emulate($code, $tokens);
+            }
+        }
+
+        return $tokens;
     }
 
-    /**
+    private function isHeredocNowdocEmulationNeeded(string $code): bool
-     * Prepares code for emulation. If nothing has to be emulated null is returned.
+    {
-     *
+        // skip version where this works without emulation
-     * @param string $code
-     * @return null|string
-     */
-    private function prepareCode(string $code) {
         if (version_compare(\PHP_VERSION, self::PHP_7_3, '>=')) {
-            return null;
+            return false;
         }
 
-        if (strpos($code, '<<<') === false) {
+        return strpos($code, '<<<') !== false;
-            // Definitely doesn't contain heredoc/nowdoc
+    }
-            return null;
+
+    private function processHeredocNowdoc(string $code): string
+    {
+        if ($this->isHeredocNowdocEmulationNeeded($code) === false) {
+            return $code;
         }
 
-        $flexibleDocStringRegex = <<<'REGEX'
+        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
-/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
-(?:.*\r?\n)*?
-(?<indentation>\h*)\2(?![a-zA-Z_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
-REGEX;
-        if (!preg_match_all($flexibleDocStringRegex, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
             // No heredoc/nowdoc found
-            return null;
+            return $code;
         }
 
         // Keep track of how much we need to adjust string offsets due to the modifications we
@@ -93,52 +125,56 @@ REGEX;
             }
         }
 
-        if (empty($this->patches)) {
-            // We did not end up emulating anything
-            return null;
-        }
-
         return $code;
     }
 
-    private function fixupTokens() {
+    private function isEmulationNeeded(string $code): bool
-        assert(count($this->patches) > 0);
+    {
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                return true;
+            }
+        }
+
+        return $this->isHeredocNowdocEmulationNeeded($code);
+    }
+
+    private function fixupTokens(array $tokens): array
+    {
+        if (\count($this->patches) === 0) {
+            return $tokens;
+        }
 
         // Load first patch
         $patchIdx = 0;
+
         list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
 
         // We use a manual loop over the tokens, because we modify the array on the fly
         $pos = 0;
-        for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
+        for ($i = 0, $c = \count($tokens); $i < $c; $i++) {
-            $token = $this->tokens[$i];
+            $token = $tokens[$i];
-            if (\is_string($token)) {
+            $len = \strlen($token->value);
-                // We assume that patches don't apply to string tokens
-                $pos += \strlen($token);
-                continue;
-            }
-
-            $len = \strlen($token[1]);
             $posDelta = 0;
             while ($patchPos >= $pos && $patchPos < $pos + $len) {
                 $patchTextLen = \strlen($patchText);
                 if ($patchType === 'remove') {
                     if ($patchPos === $pos && $patchTextLen === $len) {
                         // Remove token entirely
-                        array_splice($this->tokens, $i, 1, []);
+                        array_splice($tokens, $i, 1, []);
                         $i--;
                         $c--;
                     } else {
                         // Remove from token string
-                        $this->tokens[$i][1] = substr_replace(
+                        $tokens[$i]->value = substr_replace(
-                            $token[1], '', $patchPos - $pos + $posDelta, $patchTextLen
+                            $token->value, '', $patchPos - $pos + $posDelta, $patchTextLen
                         );
                         $posDelta -= $patchTextLen;
                     }
                 } elseif ($patchType === 'add') {
                     // Insert into the token string
-                    $this->tokens[$i][1] = substr_replace(
+                    $tokens[$i]->value = substr_replace(
-                        $token[1], $patchText, $patchPos - $pos + $posDelta, 0
+                        $token->value, $patchText, $patchPos - $pos + $posDelta, 0
                     );
                     $posDelta += $patchTextLen;
                 } else {
@@ -149,21 +185,30 @@ REGEX;
                 $patchIdx++;
                 if ($patchIdx >= \count($this->patches)) {
                     // No more patches, we're done
-                    return;
+                    break 2;
                 }
 
                 list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
 
                 // Multiple patches may apply to the same token. Reload the current one to check
                 // If the new patch applies
-                $token = $this->tokens[$i];
+                $token = $tokens[$i];
             }
 
             $pos += $len;
         }
 
-        // A patch did not apply
+        // To retain a minimum amount of sanity, recompute lines and offsets in a separate loop.
-        assert(false);
+        $pos = 0;
+        $line = 1;
+        foreach ($tokens as $token) {
+            $token->filePos = $pos;
+            $token->line = $line;
+            $pos += \strlen($token->value);
+            $line += \substr_count($token->value, "\n");
+        }
+
+        return $tokens;
     }
 
     /**
@@ -200,4 +245,4 @@ REGEX;
             $error->setAttributes($attrs);
         }
     }
-}
+}

lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php

@@ -0,0 +1,40 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;
+
+final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return strpos($code, '??=') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            if (isset($tokens[$i + 1])) {
+                $token = $tokens[$i];
+                if ($token->id === Tokens::T_COALESCE && $tokens[$i + 1]->value === '=') {
+                    array_splice($tokens, $i, 2, [
+                        new Token(Tokens::T_COALESCE_EQUAL, '??=', $token->line, $token->filePos),
+                    ]);
+                    $c--;
+                    continue;
+                }
+            }
+        }
+
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php

@@ -0,0 +1,56 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;
+
+final class FnTokenEmulator implements TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return strpos($code, 'fn') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        foreach ($tokens as $i => $token) {
+            if ($token->id === Tokens::T_STRING && $token->value === 'fn') {
+                $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $i);
+                if ($previousNonSpaceToken !== null
+                        && $previousNonSpaceToken->id === Tokens::T_OBJECT_OPERATOR) {
+                    continue;
+                }
+
+                $token->id = Tokens::T_FN;
+            }
+        }
+
+        return $tokens;
+    }
+
+    /**
+     * @param Token[] $tokens
+     * @return Token|null
+     */
+    private function getPreviousNonSpaceToken(array $tokens, int $start)
+    {
+        for ($i = $start - 1; $i >= 0; --$i) {
+            if ($tokens[$i]->id === Tokens::T_WHITESPACE) {
+                continue;
+            }
+
+            return $tokens[$i];
+        }
+
+        return null;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -0,0 +1,100 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;
+
+final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
+{
+    const BIN = '(?:0b[01]+(?:_[01]+)*)';
+    const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
+    const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
+    const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
+    const EXP = '(?:e[+-]?' . self::DEC . ')';
+    const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
+    const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return preg_match('~[0-9a-f]_[0-9a-f]~i', $code) !== false;
+    }
+
+    /**
+     * @param Token[] $tokens
+     * @return Token[]
+     */
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            $tokenLen = \strlen($token->value);
+
+            if ($token->id !== Tokens::T_LNUMBER && $token->id !== Tokens::T_DNUMBER) {
+                continue;
+            }
+
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $token->filePos);
+            assert($res, "No number at number token position");
+
+            $match = $matches[0];
+            $matchLen = \strlen($match);
+            if ($matchLen === $tokenLen) {
+                // Original token already holds the full number.
+                continue;
+            }
+
+            $tokenKind = $this->resolveIntegerOrFloatToken($match);
+            $newTokens = [new Token($tokenKind, $match, $token->line, $token->filePos)];
+
+            $numTokens = 1;
+            $len = $tokenLen;
+            while ($matchLen > $len) {
+                $nextToken = $tokens[$i + $numTokens];
+                $nextTokenLen = \strlen($nextToken->value);
+
+                $numTokens++;
+                if ($matchLen < $len + $nextTokenLen) {
+                    // Split trailing characters into a partial token.
+                    $partialText = substr($nextToken->value, $matchLen - $len);
+                    $newTokens[] = new Token(
+                        $nextToken->id, $partialText, $nextToken->line, $nextToken->filePos
+                    );
+                    break;
+                }
+
+                $len += $nextTokenLen;
+            }
+
+            array_splice($tokens, $i, $numTokens, $newTokens);
+            $c -= $numTokens - \count($newTokens);
+        }
+
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int
+    {
+        $str = str_replace('_', '', $str);
+
+        if (stripos($str, '0b') === 0) {
+            $num = bindec($str);
+        } elseif (stripos($str, '0x') === 0) {
+            $num = hexdec($str);
+        } elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
+            $num = octdec($str);
+        } else {
+            $num = +$str;
+        }
+
+        return is_float($num) ? Tokens::T_DNUMBER : Tokens::T_LNUMBER;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/TokenEmulatorInterface.php

@@ -0,0 +1,17 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Token;
+
+/** @internal */
+interface TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code): bool;
+
+    /**
+     * @param Token[] $tokens
+     * @return Token[]
+     */
+    public function emulate(string $code, array $tokens): array;
+}

lib/PhpParser/Node/Arg.php

@@ -22,7 +22,7 @@ class Arg extends NodeAbstract
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $value, bool $byRef = false, bool $unpack = false, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
         $this->byRef = $byRef;
         $this->unpack = $unpack;

lib/PhpParser/Node/Const_.php

@@ -22,7 +22,7 @@ class Const_ extends NodeAbstract
      * @param array             $attributes Additional attributes
      */
     public function __construct($name, Expr $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
         $this->value = $value;
     }

lib/PhpParser/Node/Expr/ArrayDimFetch.php

@@ -19,7 +19,7 @@ class ArrayDimFetch extends Expr
      * @param array     $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $dim = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->dim = $dim;
     }

lib/PhpParser/Node/Expr/ArrayItem.php

@@ -12,6 +12,8 @@ class ArrayItem extends Expr
     public $value;
     /** @var bool Whether to assign by reference */
     public $byRef;
+    /** @var bool Whether to unpack the argument */
+    public $unpack;
 
     /**
      * Constructs an array item node.
@@ -21,17 +23,18 @@ class ArrayItem extends Expr
      * @param bool      $byRef      Whether to assign by reference
      * @param array     $attributes Additional attributes
      */
-    public function __construct(Expr $value, Expr $key = null, bool $byRef = false, array $attributes = []) {
+    public function __construct(Expr $value, Expr $key = null, bool $byRef = false, array $attributes = [], bool $unpack = false) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->key = $key;
         $this->value = $value;
         $this->byRef = $byRef;
+        $this->unpack = $unpack;
     }
 
     public function getSubNodeNames() : array {
-        return ['key', 'value', 'byRef'];
+        return ['key', 'value', 'byRef', 'unpack'];
     }
-    
+
     public function getType() : string {
         return 'Expr_ArrayItem';
     }

lib/PhpParser/Node/Expr/Array_.php

@@ -20,7 +20,7 @@ class Array_ extends Expr
      * @param array       $attributes Additional attributes
      */
     public function __construct(array $items = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->items = $items;
     }
 

lib/PhpParser/Node/Expr/ArrowFunction.php

@@ -0,0 +1,71 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node\Expr;
+
+use PhpParser\Node;
+use PhpParser\Node\Expr;
+use PhpParser\Node\FunctionLike;
+
+class ArrowFunction extends Expr implements FunctionLike
+{
+    /** @var bool */
+    public $static;
+
+    /** @var bool */
+    public $byRef;
+
+    /** @var Node\Param[] */
+    public $params = [];
+
+    /** @var null|Node\Identifier|Node\Name|Node\NullableType */
+    public $returnType;
+
+    /** @var Expr */
+    public $expr;
+
+    /**
+     * @param array $subNodes   Array of the following optional subnodes:
+     *                          'static'     => false   : Whether the closure is static
+     *                          'byRef'      => false   : Whether to return by reference
+     *                          'params'     => array() : Parameters
+     *                          'returnType' => null    : Return type
+     *                          'expr'       => Expr    : Expression body
+     * @param array $attributes Additional attributes
+     */
+    public function __construct(array $subNodes = [], array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->static = $subNodes['static'] ?? false;
+        $this->byRef = $subNodes['byRef'] ?? false;
+        $this->params = $subNodes['params'] ?? [];
+        $returnType = $subNodes['returnType'] ?? null;
+        $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
+        $this->expr = $subNodes['expr'] ?? null;
+    }
+
+    public function getSubNodeNames() : array {
+        return ['static', 'byRef', 'params', 'returnType', 'expr'];
+    }
+
+    public function returnsByRef() : bool {
+        return $this->byRef;
+    }
+
+    public function getParams() : array {
+        return $this->params;
+    }
+
+    public function getReturnType() {
+        return $this->returnType;
+    }
+
+    /**
+     * @return Node\Stmt\Return_[]
+     */
+    public function getStmts() : array {
+        return [new Node\Stmt\Return_($this->expr)];
+    }
+
+    public function getType() : string {
+        return 'Expr_ArrowFunction';
+    }
+}

lib/PhpParser/Node/Expr/Assign.php

@@ -19,7 +19,7 @@ class Assign extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }

lib/PhpParser/Node/Expr/AssignOp.php

@@ -19,7 +19,7 @@ abstract class AssignOp extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }

lib/PhpParser/Node/Expr/AssignOp/Coalesce.php

@@ -0,0 +1,12 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node\Expr\AssignOp;
+
+use PhpParser\Node\Expr\AssignOp;
+
+class Coalesce extends AssignOp
+{
+    public function getType() : string {
+        return 'Expr_AssignOp_Coalesce';
+    }
+}

lib/PhpParser/Node/Expr/AssignRef.php

@@ -19,7 +19,7 @@ class AssignRef extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }

lib/PhpParser/Node/Expr/BinaryOp.php

@@ -19,7 +19,7 @@ abstract class BinaryOp extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $left, Expr $right, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->left = $left;
         $this->right = $right;
     }

lib/PhpParser/Node/Expr/BitwiseNot.php

@@ -16,7 +16,7 @@ class BitwiseNot extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/BooleanNot.php

@@ -16,7 +16,7 @@ class BooleanNot extends Expr
      * @param array               $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Cast.php

@@ -16,7 +16,7 @@ abstract class Cast extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Cast/Double.php

@@ -6,6 +6,11 @@ use PhpParser\Node\Expr\Cast;
 
 class Double extends Cast
 {
+    // For use in "kind" attribute
+    const KIND_DOUBLE = 1; // "double" syntax
+    const KIND_FLOAT = 2;  // "float" syntax
+    const KIND_REAL = 3; // "real" syntax
+
     public function getType() : string {
         return 'Expr_Cast_Double';
     }

lib/PhpParser/Node/Expr/ClassConstFetch.php

@@ -21,7 +21,7 @@ class ClassConstFetch extends Expr
      * @param array                   $attributes Additional attributes
      */
     public function __construct($class, $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->class = $class;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
     }

lib/PhpParser/Node/Expr/Clone_.php

@@ -16,7 +16,7 @@ class Clone_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Closure.php

@@ -34,7 +34,7 @@ class Closure extends Expr implements FunctionLike
      * @param array $attributes Additional attributes
      */
     public function __construct(array $subNodes = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->static = $subNodes['static'] ?? false;
         $this->byRef = $subNodes['byRef'] ?? false;
         $this->params = $subNodes['params'] ?? [];

lib/PhpParser/Node/Expr/ClosureUse.php

@@ -19,7 +19,7 @@ class ClosureUse extends Expr
      * @param array         $attributes Additional attributes
      */
     public function __construct(Expr\Variable $var, bool $byRef = false, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->byRef = $byRef;
     }

lib/PhpParser/Node/Expr/ConstFetch.php

@@ -17,7 +17,7 @@ class ConstFetch extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Name $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = $name;
     }
 

lib/PhpParser/Node/Expr/Empty_.php

@@ -16,7 +16,7 @@ class Empty_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Error.php

@@ -18,7 +18,7 @@ class Error extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
     }
 
     public function getSubNodeNames() : array {

lib/PhpParser/Node/Expr/ErrorSuppress.php

@@ -16,7 +16,7 @@ class ErrorSuppress extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Eval_.php

@@ -16,7 +16,7 @@ class Eval_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Exit_.php

@@ -20,7 +20,7 @@ class Exit_ extends Expr
      * @param array                    $attributes Additional attributes
      */
     public function __construct(Expr $expr = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/FuncCall.php

@@ -20,7 +20,7 @@ class FuncCall extends Expr
      * @param array          $attributes Additional attributes
      */
     public function __construct($name, array $args = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = $name;
         $this->args = $args;
     }

lib/PhpParser/Node/Expr/Include_.php

@@ -24,7 +24,7 @@ class Include_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, int $type, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
         $this->type = $type;
     }

lib/PhpParser/Node/Expr/Instanceof_.php

@@ -20,7 +20,7 @@ class Instanceof_ extends Expr
      * @param array     $attributes Additional attributes
      */
     public function __construct(Expr $expr, $class, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
         $this->class = $class;
     }

lib/PhpParser/Node/Expr/Isset_.php

@@ -16,7 +16,7 @@ class Isset_ extends Expr
      * @param array  $attributes Additional attributes
      */
     public function __construct(array $vars, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->vars = $vars;
     }
 

lib/PhpParser/Node/Expr/List_.php

@@ -16,7 +16,7 @@ class List_ extends Expr
      * @param array              $attributes Additional attributes
      */
     public function __construct(array $items, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->items = $items;
     }
 

lib/PhpParser/Node/Expr/MethodCall.php

@@ -24,7 +24,7 @@ class MethodCall extends Expr
      * @param array                  $attributes Additional attributes
      */
     public function __construct(Expr $var, $name, array $args = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
         $this->args = $args;

lib/PhpParser/Node/Expr/New_.php

@@ -20,7 +20,7 @@ class New_ extends Expr
      * @param array                           $attributes Additional attributes
      */
     public function __construct($class, array $args = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->class = $class;
         $this->args = $args;
     }

lib/PhpParser/Node/Expr/PostDec.php

@@ -16,7 +16,7 @@ class PostDec extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
     }
 

lib/PhpParser/Node/Expr/PostInc.php

@@ -16,7 +16,7 @@ class PostInc extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
     }
 

lib/PhpParser/Node/Expr/PreDec.php

@@ -16,7 +16,7 @@ class PreDec extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
     }
 

lib/PhpParser/Node/Expr/PreInc.php

@@ -16,7 +16,7 @@ class PreInc extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
     }
 

lib/PhpParser/Node/Expr/Print_.php

@@ -16,7 +16,7 @@ class Print_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/PropertyFetch.php

@@ -20,7 +20,7 @@ class PropertyFetch extends Expr
      * @param array                  $attributes Additional attributes
      */
     public function __construct(Expr $var, $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
     }

lib/PhpParser/Node/Expr/ShellExec.php

@@ -16,7 +16,7 @@ class ShellExec extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(array $parts, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->parts = $parts;
     }
 

lib/PhpParser/Node/Expr/StaticCall.php

@@ -24,7 +24,7 @@ class StaticCall extends Expr
      * @param array                  $attributes Additional attributes
      */
     public function __construct($class, $name, array $args = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->class = $class;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
         $this->args = $args;

lib/PhpParser/Node/Expr/StaticPropertyFetch.php

@@ -21,7 +21,7 @@ class StaticPropertyFetch extends Expr
      * @param array                         $attributes Additional attributes
      */
     public function __construct($class, $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->class = $class;
         $this->name = \is_string($name) ? new VarLikeIdentifier($name) : $name;
     }

lib/PhpParser/Node/Expr/Ternary.php

@@ -22,7 +22,7 @@ class Ternary extends Expr
      * @param array                    $attributes Additional attributes
      */
     public function __construct(Expr $cond, $if, Expr $else, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->cond = $cond;
         $this->if = $if;
         $this->else = $else;

lib/PhpParser/Node/Expr/UnaryMinus.php

@@ -16,7 +16,7 @@ class UnaryMinus extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/UnaryPlus.php

@@ -16,7 +16,7 @@ class UnaryPlus extends Expr
      * @param array               $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Variable.php

@@ -16,7 +16,7 @@ class Variable extends Expr
      * @param array                      $attributes Additional attributes
      */
     public function __construct($name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = $name;
     }
 

lib/PhpParser/Node/Expr/YieldFrom.php

@@ -16,7 +16,7 @@ class YieldFrom extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Yield_.php

@@ -19,7 +19,7 @@ class Yield_ extends Expr
      * @param array     $attributes Additional attributes
      */
     public function __construct(Expr $value = null, Expr $key = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->key = $key;
         $this->value = $value;
     }

lib/PhpParser/Node/Identifier.php

@@ -25,7 +25,7 @@ class Identifier extends NodeAbstract
      * @param array  $attributes Additional attributes
      */
     public function __construct(string $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = $name;
     }
 

lib/PhpParser/Node/Name.php

@@ -6,9 +6,7 @@ use PhpParser\NodeAbstract;
 
 class Name extends NodeAbstract
 {
-    /**
+    /** @var string[] Parts of the name */
-     * @var string[] Parts of the name
-     */
     public $parts;
 
     private static $specialClassNames = [
@@ -24,7 +22,7 @@ class Name extends NodeAbstract
      * @param array                $attributes Additional attributes
      */
     public function __construct($name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->parts = self::prepareName($name);
     }
 
@@ -237,7 +235,7 @@ class Name extends NodeAbstract
             'Expected string, array of parts or Name instance'
         );
     }
-    
+
     public function getType() : string {
         return 'Name';
     }

lib/PhpParser/Node/NullableType.php

@@ -16,7 +16,7 @@ class NullableType extends NodeAbstract
      * @param array                  $attributes Additional attributes
      */
     public function __construct($type, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->type = \is_string($type) ? new Identifier($type) : $type;
     }
 

lib/PhpParser/Node/Param.php

@@ -6,7 +6,7 @@ use PhpParser\NodeAbstract;
 
 class Param extends NodeAbstract
 {
-    /** @var null|Identifier|Name|NullableType Typehint */
+    /** @var null|Identifier|Name|NullableType Type declaration */
     public $type;
     /** @var bool Whether parameter is passed by reference */
     public $byRef;
@@ -20,18 +20,18 @@ class Param extends NodeAbstract
     /**
      * Constructs a parameter node.
      *
-     * @param Expr\Variable|Expr\Error      $var        Parameter variable
+     * @param Expr\Variable|Expr\Error                 $var        Parameter variable
-     * @param null|Expr                     $default    Default value
+     * @param null|Expr                                $default    Default value
-     * @param null|string|Name|NullableType $type       Typehint
+     * @param null|string|Identifier|Name|NullableType $type       Type declaration
-     * @param bool                          $byRef      Whether is passed by reference
+     * @param bool                                     $byRef      Whether is passed by reference
-     * @param bool                          $variadic   Whether this is a variadic argument
+     * @param bool                                     $variadic   Whether this is a variadic argument
-     * @param array                         $attributes Additional attributes
+     * @param array                                    $attributes Additional attributes
      */
     public function __construct(
         $var, Expr $default = null, $type = null,
         bool $byRef = false, bool $variadic = false, array $attributes = []
     ) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->type = \is_string($type) ? new Identifier($type) : $type;
         $this->byRef = $byRef;
         $this->variadic = $variadic;
@@ -42,7 +42,7 @@ class Param extends NodeAbstract
     public function getSubNodeNames() : array {
         return ['type', 'byRef', 'variadic', 'var', 'default'];
     }
-    
+
     public function getType() : string {
         return 'Param';
     }

lib/PhpParser/Node/Scalar/DNumber.php

@@ -16,7 +16,7 @@ class DNumber extends Scalar
      * @param array $attributes Additional attributes
      */
     public function __construct(float $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
     }
 
@@ -34,6 +34,8 @@ class DNumber extends Scalar
      * @return float The parsed number
      */
     public static function parse(string $str) : float {
+        $str = str_replace('_', '', $str);
+
         // if string contains any of .eE just cast it to float
         if (false !== strpbrk($str, '.eE')) {
             return (float) $str;

lib/PhpParser/Node/Scalar/Encapsed.php

@@ -17,7 +17,7 @@ class Encapsed extends Scalar
      * @param array  $attributes Additional attributes
      */
     public function __construct(array $parts, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->parts = $parts;
     }
 

lib/PhpParser/Node/Scalar/EncapsedStringPart.php

@@ -16,7 +16,7 @@ class EncapsedStringPart extends Scalar
      * @param array  $attributes Additional attributes
      */
     public function __construct(string $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
     }
 

lib/PhpParser/Node/Scalar/LNumber.php

@@ -23,7 +23,7 @@ class LNumber extends Scalar
      * @param array $attributes Additional attributes
      */
     public function __construct(int $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
     }
 
@@ -41,6 +41,8 @@ class LNumber extends Scalar
      * @return LNumber The constructed LNumber, including kind attribute
      */
     public static function fromString(string $str, array $attributes = [], bool $allowInvalidOctal = false) : LNumber {
+        $str = str_replace('_', '', $str);
+
         if ('0' !== $str[0] || '0' === $str) {
             $attributes['kind'] = LNumber::KIND_DEC;
             return new LNumber((int) $str, $attributes);

lib/PhpParser/Node/Scalar/MagicConst.php

@@ -12,7 +12,7 @@ abstract class MagicConst extends Scalar
      * @param array $attributes Additional attributes
      */
     public function __construct(array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
     }
 
     public function getSubNodeNames() : array {

lib/PhpParser/Node/Scalar/String_.php

@@ -34,7 +34,7 @@ class String_ extends Scalar
      * @param array  $attributes Additional attributes
      */
     public function __construct(string $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
     }
 

lib/PhpParser/Node/Stmt/Break_.php

@@ -16,7 +16,7 @@ class Break_ extends Node\Stmt
      * @param array          $attributes Additional attributes
      */
     public function __construct(Node\Expr $num = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->num = $num;
     }
 

lib/PhpParser/Node/Stmt/Case_.php

@@ -6,7 +6,7 @@ use PhpParser\Node;
 
 class Case_ extends Node\Stmt
 {
-    /** @var null|Node\Expr $cond Condition (null for default) */
+    /** @var null|Node\Expr Condition (null for default) */
     public $cond;
     /** @var Node\Stmt[] Statements */
     public $stmts;
@@ -19,7 +19,7 @@ class Case_ extends Node\Stmt
      * @param array          $attributes Additional attributes
      */
     public function __construct($cond, array $stmts = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->cond = $cond;
         $this->stmts = $stmts;
     }

lib/PhpParser/Node/Stmt/Catch_.php

@@ -25,7 +25,7 @@ class Catch_ extends Node\Stmt
     public function __construct(
         array $types, Expr\Variable $var, array $stmts = [], array $attributes = []
     ) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->types = $types;
         $this->var = $var;
         $this->stmts = $stmts;

lib/PhpParser/Node/Stmt/ClassConst.php

@@ -19,7 +19,7 @@ class ClassConst extends Node\Stmt
      * @param array         $attributes Additional attributes
      */
     public function __construct(array $consts, int $flags = 0, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->flags = $flags;
         $this->consts = $consts;
     }

lib/PhpParser/Node/Stmt/ClassMethod.php

@@ -51,7 +51,7 @@ class ClassMethod extends Node\Stmt implements FunctionLike
      * @param array $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->flags = $subNodes['flags'] ?? $subNodes['type'] ?? 0;
         $this->byRef = $subNodes['byRef'] ?? false;
         $this->name = \is_string($name) ? new Node\Identifier($name) : $name;

lib/PhpParser/Node/Stmt/Class_.php

@@ -35,7 +35,7 @@ class Class_ extends ClassLike
      * @param array       $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->flags = $subNodes['flags'] ?? $subNodes['type'] ?? 0;
         $this->name = \is_string($name) ? new Node\Identifier($name) : $name;
         $this->extends = $subNodes['extends'] ?? null;

lib/PhpParser/Node/Stmt/Const_.php

@@ -16,7 +16,7 @@ class Const_ extends Node\Stmt
      * @param array         $attributes Additional attributes
      */
     public function __construct(array $consts, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->consts = $consts;
     }
 

lib/PhpParser/Node/Stmt/Continue_.php

@@ -16,7 +16,7 @@ class Continue_ extends Node\Stmt
      * @param array          $attributes Additional attributes
      */
     public function __construct(Node\Expr $num = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->num = $num;
     }
 

lib/PhpParser/Node/Stmt/DeclareDeclare.php

@@ -19,7 +19,7 @@ class DeclareDeclare extends Node\Stmt
      * @param array                  $attributes Additional attributes
      */
     public function __construct($key, Node\Expr $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->key = \is_string($key) ? new Node\Identifier($key) : $key;
         $this->value = $value;
     }

lib/PhpParser/Node/Stmt/Declare_.php

@@ -19,7 +19,7 @@ class Declare_ extends Node\Stmt
      * @param array            $attributes Additional attributes
      */
     public function __construct(array $declares, array $stmts = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->declares = $declares;
         $this->stmts = $stmts;
     }

lib/PhpParser/Node/Stmt/Do_.php

@@ -19,7 +19,7 @@ class Do_ extends Node\Stmt
      * @param array       $attributes Additional attributes
      */
     public function __construct(Node\Expr $cond, array $stmts = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->cond = $cond;
         $this->stmts = $stmts;
     }

lib/PhpParser/Node/Stmt/Echo_.php

@@ -16,7 +16,7 @@ class Echo_ extends Node\Stmt
      * @param array       $attributes Additional attributes
      */
     public function __construct(array $exprs, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->exprs = $exprs;
     }
 

lib/PhpParser/Node/Stmt/ElseIf_.php

@@ -19,7 +19,7 @@ class ElseIf_ extends Node\Stmt
      * @param array       $attributes Additional attributes
      */
     public function __construct(Node\Expr $cond, array $stmts = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->cond = $cond;
         $this->stmts = $stmts;
     }

lib/PhpParser/Node/Stmt/Else_.php

@@ -16,7 +16,7 @@ class Else_ extends Node\Stmt
      * @param array       $attributes Additional attributes
      */
     public function __construct(array $stmts = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->stmts = $stmts;
     }
 

lib/PhpParser/Node/Stmt/Expression.php

@@ -19,7 +19,7 @@ class Expression extends Node\Stmt
      * @param array     $attributes Additional attributes
      */
     public function __construct(Node\Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Stmt/Finally_.php

@@ -16,7 +16,7 @@ class Finally_ extends Node\Stmt
      * @param array       $attributes Additional attributes
      */
     public function __construct(array $stmts = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->stmts = $stmts;
     }
 

lib/PhpParser/Node/Stmt/For_.php

@@ -26,7 +26,7 @@ class For_ extends Node\Stmt
      * @param array $attributes Additional attributes
      */
     public function __construct(array $subNodes = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->init = $subNodes['init'] ?? [];
         $this->cond = $subNodes['cond'] ?? [];
         $this->loop = $subNodes['loop'] ?? [];

lib/PhpParser/Node/Stmt/Foreach_.php

@@ -29,7 +29,7 @@ class Foreach_ extends Node\Stmt
      * @param array     $attributes Additional attributes
      */
     public function __construct(Node\Expr $expr, Node\Expr $valueVar, array $subNodes = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
         $this->keyVar = $subNodes['keyVar'] ?? null;
         $this->byRef = $subNodes['byRef'] ?? false;