Release PHP-Parser 4.5.0

Similar to the previous commit: list() syntactically accepts any
expression and non-variables are compile-time errors. The special
case of ($a) ends up being legal by accident.

.gitattributes

@@ -0,0 +1,9 @@
+/doc export-ignore
+/test export-ignore
+/test_old export-ignore
+.gitattributes export-ignore
+.gitignore export-ignore
+.travis.yml export-ignore
+CHANGELOG.md export-ignore
+phpunit.xml.dist export-ignore
+UPGRADE-*.md export-ignore

.gitignore

@@ -2,3 +2,4 @@ vendor/
 composer.lock
 grammar/kmyacc.exe
 grammar/y.output
+.phpunit.result.cache

.travis.yml

@@ -1,6 +1,5 @@
 language: php
 dist: xenial
-sudo: false
 
 cache:
   directories:
@@ -11,11 +10,17 @@ php:
   - 7.1
   - 7.2
   - 7.3
+  - 7.4
   - nightly
 
 install:
   - if [ $TRAVIS_PHP_VERSION = '7.0' ]; then composer require satooshi/php-coveralls '~1.0'; fi
-  - composer install --prefer-dist
+  - |
+    if [ $TRAVIS_PHP_VERSION = 'nightly' ]; then
+      composer install --prefer-dist --ignore-platform-reqs;
+    else
+      composer install --prefer-dist;
+    fi
 
 matrix:
   allow_failures:

CHANGELOG.md

@@ -1,15 +1,123 @@
-Version 4.1.2-dev
+Version 4.5.1-dev
 -----------------
 
 Nothing yet.
 
+Version 4.5.0 (2020-06-03)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for the mixed type. This means `mixed` types are now parsed as an
+  `Identifier` rather than a `Name`.
+* [PHP 8.0] Added support for catching without capturing the exception. This means that
+  `Catch_::$var` may now be null.
+
+Version 4.4.0 (2020-04-10)
+--------------------------
+
+### Added
+
+* Added support for passing union types in builders.
+* Added end line, token position and file position information for comments.
+* Added `getProperty()` method to `ClassLike` nodes.
+
+### Fixed
+
+* Fixed generation of invalid code when using the formatting preserving pretty printer, and
+  inserting code next to certain nop statements. The formatting is still ugly though.
+* `getDocComment()` no longer requires that the very last comment before a node be a doc comment.
+  There may not be non-doc comments between the doc comment and the declaration.
+* Allowed arbitrary expressions in `isset()` and `list()`, rather than just variables.
+  In particular, this allows `isset(($x))`, which is legal PHP code.
+* [PHP 8.0] Add support for [variable syntax tweaks RFC](https://wiki.php.net/rfc/variable_syntax_tweaks).
+
+Version 4.3.0 (2019-11-08)
+--------------------------
+
+### Added
+
+* [PHP 8.0] Added support for union types using a new `UnionType` node.
+
+Version 4.2.5 (2019-10-25)
+--------------------------
+
+### Changed
+
+* Tests and documentation are no longer included in source archives. They can still be accessed
+  by cloning the repository.
+* php-yacc is now used to generate the parser. This has no impact on users of the library.
+
+Version 4.2.4 (2019-09-01)
+--------------------------
+
+### Added
+
+* Added getProperties(), getConstants() and getTraitUses() to ClassLike. (#629, #630)
+
+### Fixed
+
+* Fixed flexible heredoc emulation to check for digits after the end label. This synchronizes
+  behavior with the upcoming PHP 7.3.10 release.
+
+Version 4.2.3 (2019-08-12)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for numeric literal separators. (#615)
+
+### Fixed
+
+* Fixed resolution of return types for arrow functions. (#613)
+* Fixed compatibility with PHP 7.4.
+
+Version 4.2.2 (2019-05-25)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for arrow functions using a new `Expr\ArrowFunction` node. (#602)
+* [PHP 7.4] Add support for array spreads, using a new `unpack` subnode on `ArrayItem`. (#609)
+* Added support for inserting into empty list nodes in the formatting preserving pretty printer.
+
+### Changed
+
+* `php-parse` will now print messages to stderr, so that stdout only contains the actual result of
+  the operation (such as a JSON dump). (#605)
+
+### Fixed
+
+* Fixed attribute assignment for zero-length nop statements, and a related assertion failure in
+  the formatting-preserving pretty printer. (#589)
+
+Version 4.2.1 (2019-02-16)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for `??=` operator through a new `AssignOp\Coalesce` node. (#575)
+
+Version 4.2.0 (2019-01-12)
+--------------------------
+
+### Added
+
+* [PHP 7.4] Add support for typed properties through a new `type` subnode of `Stmt\Property`.
+  Additionally `Builder\Property` now has a `setType()` method. (#567)
+* Add `kind` attribute to `Cast\Double_`, which allows to distinguish between `(float)`,
+  `(double)` and `(real)`. The form of the cast will be preserved by the pretty printer. (#565)
+
+### Fixed
+
+* Remove assertion when pretty printing anonymous class with a name (#554).
+
 Version 4.1.1 (2018-12-26)
 --------------------------
 
 ### Fixed
 
 * Fix "undefined offset" notice when parsing specific malformed code (#551).
-* Remove assertion when pretty printing anonymous class with a name (#554).
 
 ### Added
 

README.md

@@ -3,10 +3,10 @@ PHP Parser
 
 [![Build Status](https://travis-ci.org/nikic/PHP-Parser.svg?branch=master)](https://travis-ci.org/nikic/PHP-Parser) [![Coverage Status](https://coveralls.io/repos/github/nikic/PHP-Parser/badge.svg?branch=master)](https://coveralls.io/github/nikic/PHP-Parser?branch=master)
 
-This is a PHP 5.2 to PHP 7.3 parser written in PHP. Its purpose is to simplify static code analysis and
+This is a PHP 5.2 to PHP 7.4 parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
-[**Documentation for version 4.x**][doc_master] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 7.3).
+[**Documentation for version 4.x**][doc_master] (stable; for running on PHP >= 7.0; for parsing PHP 5.2 to PHP 7.4).
 
 [Documentation for version 3.x][doc_3_x] (unsupported; for running on PHP >= 5.5; for parsing PHP 5.2 to PHP 7.2).
 

bin/php-parse

@@ -45,14 +45,15 @@ $traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);
 foreach ($files as $file) {
     if (strpos($file, '<?php') === 0) {
         $code = $file;
-        echo "====> Code $code\n";
+        fwrite(STDERR, "====> Code $code\n");
     } else {
         if (!file_exists($file)) {
-            die("File $file does not exist.\n");
+            fwrite(STDERR, "File $file does not exist.\n");
+            exit(1);
         }
 
         $code = file_get_contents($file);
-        echo "====> File $file:\n";
+        fwrite(STDERR, "====> File $file:\n");
     }
 
     if ($attributes['with-recovery']) {
@@ -60,7 +61,7 @@ foreach ($files as $file) {
         $stmts = $parser->parse($code, $errorHandler);
         foreach ($errorHandler->getErrors() as $error) {
             $message = formatErrorMessage($error, $code, $attributes['with-column-info']);
-            echo $message . "\n";
+            fwrite(STDERR, $message . "\n");
         }
         if (null === $stmts) {
             continue;
@@ -70,25 +71,26 @@ foreach ($files as $file) {
             $stmts = $parser->parse($code);
         } catch (PhpParser\Error $error) {
             $message = formatErrorMessage($error, $code, $attributes['with-column-info']);
-            die($message . "\n");
+            fwrite(STDERR, $message . "\n");
+            exit(1);
         }
     }
 
     foreach ($operations as $operation) {
         if ('dump' === $operation) {
-            echo "==> Node dump:\n";
+            fwrite(STDERR, "==> Node dump:\n");
             echo $dumper->dump($stmts, $code), "\n";
         } elseif ('pretty-print' === $operation) {
-            echo "==> Pretty print:\n";
+            fwrite(STDERR, "==> Pretty print:\n");
             echo $prettyPrinter->prettyPrintFile($stmts), "\n";
         } elseif ('json-dump' === $operation) {
-            echo "==> JSON dump:\n";
+            fwrite(STDERR, "==> JSON dump:\n");
             echo json_encode($stmts, JSON_PRETTY_PRINT), "\n";
         } elseif ('var-dump' === $operation) {
-            echo "==> var_dump():\n";
+            fwrite(STDERR, "==> var_dump():\n");
             var_dump($stmts);
         } elseif ('resolve-names' === $operation) {
-            echo "==> Resolved names.\n";
+            fwrite(STDERR, "==> Resolved names.\n");
             $stmts = $traverser->traverse($stmts);
         }
     }
@@ -104,9 +106,9 @@ function formatErrorMessage(PhpParser\Error $e, $code, $withColumnInfo) {
 
 function showHelp($error = '') {
     if ($error) {
-        echo $error . "\n\n";
+        fwrite(STDERR, $error . "\n\n");
     }
-    die(<<<OUTPUT
+    fwrite($error ? STDERR : STDOUT, <<<OUTPUT
 Usage: php-parse [operations] file1.php [file2.php ...]
    or: php-parse [operations] "<?php code"
 Turn PHP source code into an abstract syntax tree.
@@ -131,6 +133,7 @@ Example:
 
 OUTPUT
     );
+    exit($error ? 1 : 0);
 }
 
 function parseArgs($args) {

composer.json

@@ -1,8 +1,11 @@
 {
     "name": "nikic/php-parser",
-    "description": "A PHP parser written in PHP",
-    "keywords": ["php", "parser"],
     "type": "library",
+    "description": "A PHP parser written in PHP",
+    "keywords": [
+        "php",
+        "parser"
+    ],
     "license": "BSD-3-Clause",
     "authors": [
         {
@@ -14,17 +17,25 @@
         "ext-tokenizer": "*"
     },
     "require-dev": {
-        "phpunit/phpunit": "^6.5 || ^7.0"
+        "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0",
+        "ircmaxell/php-yacc": "0.0.5"
+    },
+    "extra": {
+        "branch-alias": {
+            "dev-master": "4.3-dev"
+        }
     },
     "autoload": {
         "psr-4": {
             "PhpParser\\": "lib/PhpParser"
         }
     },
-    "bin": ["bin/php-parse"],
-    "extra": {
-        "branch-alias": {
-            "dev-master": "4.1-dev"
+    "autoload-dev": {
+        "psr-4": {
+            "PhpParser\\": "test/PhpParser/"
         }
-    }
+    },
+    "bin": [
+        "bin/php-parse"
+    ]
 }

doc/0_Introduction.markdown

@@ -1,7 +1,7 @@
 Introduction
 ============
 
-This project is a PHP 5.2 to PHP 7.3 parser **written in PHP itself**.
+This project is a PHP 5.2 to PHP 7.4 parser **written in PHP itself**.
 
 What is this for?
 -----------------
@@ -26,11 +26,11 @@ programmatic PHP code analysis are incidentally PHP developers, not C developers
 What can it parse?
 ------------------
 
-The parser supports parsing PHP 5.2-7.3.
+The parser supports parsing PHP 5.2-7.4.
 
 As the parser is based on the tokens returned by `token_get_all` (which is only able to lex the PHP
 version it runs on), additionally a wrapper for emulating tokens from newer versions is provided.
-This allows to parse PHP 7.3 source code running on PHP 7.0, for example. This emulation is somewhat
+This allows to parse PHP 7.4 source code running on PHP 7.0, for example. This emulation is somewhat
 hacky and not perfect, but it should work well on any sane code.
 
 What output does it produce?
@@ -56,7 +56,7 @@ array(
 ```
 
 This matches the structure of the code: An echo statement, which takes two strings as expressions,
-with the values `Hi` and `World!`.
+with the values `Hi` and `World`.
 
 You can also see that the AST does not contain any whitespace information (but most comments are saved).
 So using it for formatting analysis is not possible.

doc/2_Usage_of_basic_components.markdown

@@ -77,7 +77,7 @@ A parser instance can be reused to parse multiple files.
 Node dumping
 ------------
 
-To dump the abstact syntax tree in human readable form, a `NodeDumper` can be used:
+To dump the abstract syntax tree in human readable form, a `NodeDumper` can be used:
 
 ```php
 <?php

doc/component/AST_builders.markdown

@@ -48,7 +48,7 @@ $node = $factory->namespace('Name\Space')
             ->makePublic()
             ->makeAbstract() // ->makeFinal()
             ->setReturnType('bool') // ->makeReturnByRef()
-            ->addParam($factory->param('someParam')->setTypeHint('SomeClass'))
+            ->addParam($factory->param('someParam')->setType('SomeClass'))
             ->setDocComment('/**
                               * This method does something.
                               *

doc/component/Walking_the_AST.markdown

@@ -295,7 +295,7 @@ Simple node finding
 
 While the node visitor mechanism is very flexible, creating a node visitor can be overly cumbersome
 for minor tasks. For this reason a `NodeFinder` is provided, which can find AST nodes that either
-satisfy a certain callback, or which are instanced of a certain node type. A couple of examples are
+satisfy a certain callback, or which are instances of a certain node type. A couple of examples are
 shown in the following:
 
 ```php
@@ -312,7 +312,7 @@ $extendingClasses = $nodeFinder->find($stmts, function(Node $node) {
         && $node->extends !== null;
 });
 
-// Find first class occuring in the AST. Returns null if no class exists.
+// Find first class occurring in the AST. Returns null if no class exists.
 $class = $nodeFinder->findFirstInstanceOf($stmts, Node\Stmt\Class_::class);
 
 // Find first class that has name $name

grammar/README.md

@@ -21,8 +21,10 @@ applied to it:
 Building the parser
 ===================
 
-In order to rebuild the parser, you need [moriyoshi's fork of kmyacc](https://github.com/moriyoshi/kmyacc-forked).
-After you compiled/installed it, run the `rebuildParsers.php` script.
+Run `php grammar/rebuildParsers.php` to rebuild the parsers. Additional options:
 
-By default only the `Parser.php` is built. If you want to additionally emit debug symbols and create `y.output`, run the
-script with `--debug`. If you want to retain the preprocessed grammar pass `--keep-tmp-grammar`.
+ * The `KMYACC` environment variable can be used to specify an alternative `kmyacc` binary.
+   By default the `phpyacc` dev dependency will be used. To use the original `kmyacc`, you
+   need to compile [moriyoshi's fork](https://github.com/moriyoshi/kmyacc-forked).
+ * The `--debug` option enables emission of debug symbols and creates the `y.output` file.
+ * The `--keep-tmp-grammar` option preserves the preprocessed grammar file.

grammar/php5.y

@@ -16,7 +16,7 @@ top_statement_list_ex:
 
 top_statement_list:
       top_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -27,7 +27,7 @@ reserved_non_modifiers:
     | T_FINALLY | T_THROW | T_USE | T_INSTEADOF | T_GLOBAL | T_VAR | T_UNSET | T_ISSET | T_EMPTY | T_CONTINUE | T_GOTO
     | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT
     | T_BREAK | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE | T_CLASS
-    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER
+    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER | T_FN
 ;
 
 semi_reserved:
@@ -160,7 +160,7 @@ inner_statement_list_ex:
 
 inner_statement_list:
       inner_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -461,7 +461,7 @@ class_statement_list_ex:
 
 class_statement_list:
       class_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -579,6 +579,7 @@ expr:
     | variable T_SL_EQUAL expr                              { $$ = Expr\AssignOp\ShiftLeft [$1, $3]; }
     | variable T_SR_EQUAL expr                              { $$ = Expr\AssignOp\ShiftRight[$1, $3]; }
     | variable T_POW_EQUAL expr                             { $$ = Expr\AssignOp\Pow       [$1, $3]; }
+    | variable T_COALESCE_EQUAL expr                        { $$ = Expr\AssignOp\Coalesce  [$1, $3]; }
     | variable T_INC                                        { $$ = Expr\PostInc[$1]; }
     | T_INC variable                                        { $$ = Expr\PreInc [$2]; }
     | variable T_DEC                                        { $$ = Expr\PostDec[$1]; }
@@ -628,7 +629,10 @@ expr:
     | T_REQUIRE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE]; }
     | T_REQUIRE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE_ONCE]; }
     | T_INT_CAST expr                                       { $$ = Expr\Cast\Int_    [$2]; }
-    | T_DOUBLE_CAST expr                                    { $$ = Expr\Cast\Double  [$2]; }
+    | T_DOUBLE_CAST expr
+          { $attrs = attributes();
+            $attrs['kind'] = $this->getFloatCastKind($1);
+            $$ = new Expr\Cast\Double($2, $attrs); }
     | T_STRING_CAST expr                                    { $$ = Expr\Cast\String_ [$2]; }
     | T_ARRAY_CAST expr                                     { $$ = Expr\Cast\Array_  [$2]; }
     | T_OBJECT_CAST expr                                    { $$ = Expr\Cast\Object_ [$2]; }
@@ -984,6 +988,7 @@ array_pair:
     | expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
     | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
     | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
+    | T_ELLIPSIS expr                                       { $$ = Expr\ArrayItem[$2, null, false, attributes(), true]; }
 ;
 
 encaps_list:

grammar/php7.y

@@ -16,7 +16,7 @@ top_statement_list_ex:
 
 top_statement_list:
       top_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -27,7 +27,7 @@ reserved_non_modifiers:
     | T_FINALLY | T_THROW | T_USE | T_INSTEADOF | T_GLOBAL | T_VAR | T_UNSET | T_ISSET | T_EMPTY | T_CONTINUE | T_GOTO
     | T_FUNCTION | T_CONST | T_RETURN | T_PRINT | T_YIELD | T_LIST | T_SWITCH | T_ENDSWITCH | T_CASE | T_DEFAULT
     | T_BREAK | T_ARRAY | T_CALLABLE | T_EXTENDS | T_IMPLEMENTS | T_NAMESPACE | T_TRAIT | T_INTERFACE | T_CLASS
-    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER
+    | T_CLASS_C | T_TRAIT_C | T_FUNC_C | T_METHOD_C | T_LINE | T_FILE | T_DIR | T_NS_C | T_HALT_COMPILER | T_FN
 ;
 
 semi_reserved:
@@ -196,7 +196,7 @@ inner_statement_list_ex:
 
 inner_statement_list:
       inner_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
@@ -232,7 +232,7 @@ non_empty_statement:
     | T_RETURN optional_expr semi                           { $$ = Stmt\Return_[$2]; }
     | T_GLOBAL global_var_list semi                         { $$ = Stmt\Global_[$2]; }
     | T_STATIC static_var_list semi                         { $$ = Stmt\Static_[$2]; }
-    | T_ECHO expr_list semi                                 { $$ = Stmt\Echo_[$2]; }
+    | T_ECHO expr_list_forbid_comma semi                    { $$ = Stmt\Echo_[$2]; }
     | T_INLINE_HTML                                         { $$ = Stmt\InlineHTML[$1]; }
     | expr semi                                             { $$ = Stmt\Expression[$1]; }
     | T_UNSET '(' variables_list ')' semi                   { $$ = Stmt\Unset_[$3]; }
@@ -269,7 +269,7 @@ name_union:
 ;
 
 catch:
-    T_CATCH '(' name_union plain_variable ')' '{' inner_statement_list '}'
+    T_CATCH '(' name_union optional_plain_variable ')' '{' inner_statement_list '}'
         { $$ = Stmt\Catch_[$3, $4, $7]; }
 ;
 
@@ -450,17 +450,18 @@ non_empty_parameter_list:
 ;
 
 parameter:
-      optional_param_type optional_ref optional_ellipsis plain_variable
+      optional_type optional_ref optional_ellipsis plain_variable
           { $$ = Node\Param[$4, null, $1, $2, $3]; $this->checkParam($$); }
-    | optional_param_type optional_ref optional_ellipsis plain_variable '=' expr
+    | optional_type optional_ref optional_ellipsis plain_variable '=' expr
           { $$ = Node\Param[$4, $6, $1, $2, $3]; $this->checkParam($$); }
-    | optional_param_type optional_ref optional_ellipsis error
+    | optional_type optional_ref optional_ellipsis error
           { $$ = Node\Param[Expr\Error[], null, $1, $2, $3]; }
 ;
 
 type_expr:
       type                                                  { $$ = $1; }
     | '?' type                                              { $$ = Node\NullableType[$2]; }
+    | union_type                                            { $$ = Node\UnionType[$1]; }
 ;
 
 type:
@@ -469,7 +470,12 @@ type:
     | T_CALLABLE                                            { $$ = Node\Identifier['callable']; }
 ;
 
-optional_param_type:
+union_type:
+      type '|' type                                         { init($1, $3); }
+    | union_type '|' type                                   { push($1, $3); }
+;
+
+optional_type:
       /* empty */                                           { $$ = null; }
     | type_expr                                             { $$ = $1; }
 ;
@@ -530,13 +536,14 @@ class_statement_list_ex:
 
 class_statement_list:
       class_statement_list_ex
-          { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes);
+          { makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
             if ($nop !== null) { $1[] = $nop; } $$ = $1; }
 ;
 
 class_statement:
-      variable_modifiers property_declaration_list ';'
-          { $$ = Stmt\Property[$1, $2]; $this->checkProperty($$, #1); }
+      variable_modifiers optional_type property_declaration_list ';'
+          { $attrs = attributes();
+            $$ = new Stmt\Property($1, $3, $attrs, $2); $this->checkProperty($$, #1); }
     | method_modifiers T_CONST class_const_list ';'
           { $$ = Stmt\ClassConst[$3, $1]; $this->checkClassConst($$, #1); }
     | method_modifiers T_FUNCTION optional_ref identifier_ex '(' parameter_list ')' optional_return_type method_body
@@ -625,10 +632,14 @@ property_declaration:
     | property_decl_name '=' expr                           { $$ = Stmt\PropertyProperty[$1, $3]; }
 ;
 
-expr_list:
+expr_list_forbid_comma:
       non_empty_expr_list no_comma                          { $$ = $1; }
 ;
 
+expr_list_allow_comma:
+      non_empty_expr_list optional_comma                    { $$ = $1; }
+;
+
 non_empty_expr_list:
       non_empty_expr_list ',' expr                          { push($1, $3); }
     | expr                                                  { init($1); }
@@ -636,7 +647,7 @@ non_empty_expr_list:
 
 for_expr:
       /* empty */                                           { $$ = array(); }
-    | expr_list                                             { $$ = $1; }
+    | expr_list_forbid_comma                                { $$ = $1; }
 ;
 
 expr:
@@ -659,6 +670,7 @@ expr:
     | variable T_SL_EQUAL expr                              { $$ = Expr\AssignOp\ShiftLeft [$1, $3]; }
     | variable T_SR_EQUAL expr                              { $$ = Expr\AssignOp\ShiftRight[$1, $3]; }
     | variable T_POW_EQUAL expr                             { $$ = Expr\AssignOp\Pow       [$1, $3]; }
+    | variable T_COALESCE_EQUAL expr                        { $$ = Expr\AssignOp\Coalesce  [$1, $3]; }
     | variable T_INC                                        { $$ = Expr\PostInc[$1]; }
     | T_INC variable                                        { $$ = Expr\PreInc [$2]; }
     | variable T_DEC                                        { $$ = Expr\PostDec[$1]; }
@@ -698,7 +710,7 @@ expr:
     | expr '?' expr ':' expr                                { $$ = Expr\Ternary[$1, $3,   $5]; }
     | expr '?' ':' expr                                     { $$ = Expr\Ternary[$1, null, $4]; }
     | expr T_COALESCE expr                                  { $$ = Expr\BinaryOp\Coalesce[$1, $3]; }
-    | T_ISSET '(' variables_list ')'                        { $$ = Expr\Isset_[$3]; }
+    | T_ISSET '(' expr_list_allow_comma ')'                 { $$ = Expr\Isset_[$3]; }
     | T_EMPTY '(' expr ')'                                  { $$ = Expr\Empty_[$3]; }
     | T_INCLUDE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_INCLUDE]; }
     | T_INCLUDE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_INCLUDE_ONCE]; }
@@ -706,7 +718,10 @@ expr:
     | T_REQUIRE expr                                        { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE]; }
     | T_REQUIRE_ONCE expr                                   { $$ = Expr\Include_[$2, Expr\Include_::TYPE_REQUIRE_ONCE]; }
     | T_INT_CAST expr                                       { $$ = Expr\Cast\Int_    [$2]; }
-    | T_DOUBLE_CAST expr                                    { $$ = Expr\Cast\Double  [$2]; }
+    | T_DOUBLE_CAST expr
+          { $attrs = attributes();
+            $attrs['kind'] = $this->getFloatCastKind($1);
+            $$ = new Expr\Cast\Double($2, $attrs); }
     | T_STRING_CAST expr                                    { $$ = Expr\Cast\String_ [$2]; }
     | T_ARRAY_CAST expr                                     { $$ = Expr\Cast\Array_  [$2]; }
     | T_OBJECT_CAST expr                                    { $$ = Expr\Cast\Object_ [$2]; }
@@ -724,6 +739,12 @@ expr:
     | T_YIELD expr                                          { $$ = Expr\Yield_[$2, null]; }
     | T_YIELD expr T_DOUBLE_ARROW expr                      { $$ = Expr\Yield_[$4, $2]; }
     | T_YIELD_FROM expr                                     { $$ = Expr\YieldFrom[$2]; }
+
+    | T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr
+          { $$ = Expr\ArrowFunction[['static' => false, 'byRef' => $2, 'params' => $4, 'returnType' => $6, 'expr' => $8]]; }
+    | T_STATIC T_FN optional_ref '(' parameter_list ')' optional_return_type T_DOUBLE_ARROW expr
+          { $$ = Expr\ArrowFunction[['static' => true, 'byRef' => $3, 'params' => $5, 'returnType' => $7, 'expr' => $9]]; }
+
     | T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type
       block_or_error
           { $$ = Expr\Closure[['static' => false, 'byRef' => $2, 'params' => $4, 'uses' => $6, 'returnType' => $7, 'stmts' => $8]]; }
@@ -783,12 +804,13 @@ name:
 class_name_reference:
       class_name                                            { $$ = $1; }
     | new_variable                                          { $$ = $1; }
+    | '(' expr ')'                                          { $$ = $2; }
     | error                                                 { $$ = Expr\Error[]; $this->errorState = 2; }
 ;
 
 class_name_or_var:
       class_name                                            { $$ = $1; }
-    | dereferencable                                        { $$ = $1; }
+    | fully_dereferencable                                  { $$ = $1; }
 ;
 
 exit_expr:
@@ -810,9 +832,20 @@ ctor_arguments:
 
 constant:
       name                                                  { $$ = Expr\ConstFetch[$1]; }
-    | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM identifier_ex
+    | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }
+    | T_FILE                                                { $$ = Scalar\MagicConst\File[]; }
+    | T_DIR                                                 { $$ = Scalar\MagicConst\Dir[]; }
+    | T_CLASS_C                                             { $$ = Scalar\MagicConst\Class_[]; }
+    | T_TRAIT_C                                             { $$ = Scalar\MagicConst\Trait_[]; }
+    | T_METHOD_C                                            { $$ = Scalar\MagicConst\Method[]; }
+    | T_FUNC_C                                              { $$ = Scalar\MagicConst\Function_[]; }
+    | T_NS_C                                                { $$ = Scalar\MagicConst\Namespace_[]; }
+;
+
+class_constant:
+      class_name_or_var T_PAAMAYIM_NEKUDOTAYIM identifier_ex
           { $$ = Expr\ClassConstFetch[$1, $3]; }
-    /* We interpret and isolated FOO:: as an unfinished class constant fetch. It could also be
+    /* We interpret an isolated FOO:: as an unfinished class constant fetch. It could also be
        an unfinished static property fetch or unfinished scoped call. */
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM error
           { $$ = Expr\ClassConstFetch[$1, new Expr\Error(stackAttributes(#3))]; $this->errorState = 2; }
@@ -832,28 +865,21 @@ dereferencable_scalar:
     | T_CONSTANT_ENCAPSED_STRING
           { $attrs = attributes(); $attrs['kind'] = strKind($1);
             $$ = new Scalar\String_(Scalar\String_::parse($1), $attrs); }
+    | '"' encaps_list '"'
+          { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
+            parseEncapsed($2, '"', true); $$ = new Scalar\Encapsed($2, $attrs); }
 ;
 
 scalar:
       T_LNUMBER                                             { $$ = $this->parseLNumber($1, attributes()); }
     | T_DNUMBER                                             { $$ = Scalar\DNumber[Scalar\DNumber::parse($1)]; }
-    | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }
-    | T_FILE                                                { $$ = Scalar\MagicConst\File[]; }
-    | T_DIR                                                 { $$ = Scalar\MagicConst\Dir[]; }
-    | T_CLASS_C                                             { $$ = Scalar\MagicConst\Class_[]; }
-    | T_TRAIT_C                                             { $$ = Scalar\MagicConst\Trait_[]; }
-    | T_METHOD_C                                            { $$ = Scalar\MagicConst\Method[]; }
-    | T_FUNC_C                                              { $$ = Scalar\MagicConst\Function_[]; }
-    | T_NS_C                                                { $$ = Scalar\MagicConst\Namespace_[]; }
     | dereferencable_scalar                                 { $$ = $1; }
     | constant                                              { $$ = $1; }
+    | class_constant                                        { $$ = $1; }
     | T_START_HEREDOC T_ENCAPSED_AND_WHITESPACE T_END_HEREDOC
           { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), true); }
     | T_START_HEREDOC T_END_HEREDOC
           { $$ = $this->parseDocString($1, '', $2, attributes(), stackAttributes(#2), true); }
-    | '"' encaps_list '"'
-          { $attrs = attributes(); $attrs['kind'] = Scalar\String_::KIND_DOUBLE_QUOTED;
-            parseEncapsed($2, '"', true); $$ = new Scalar\Encapsed($2, $attrs); }
     | T_START_HEREDOC encaps_list T_END_HEREDOC
           { $$ = $this->parseDocString($1, $2, $3, attributes(), stackAttributes(#3), true); }
 ;
@@ -863,10 +889,16 @@ optional_expr:
     | expr                                                  { $$ = $1; }
 ;
 
-dereferencable:
+fully_dereferencable:
       variable                                              { $$ = $1; }
     | '(' expr ')'                                          { $$ = $2; }
     | dereferencable_scalar                                 { $$ = $1; }
+    | class_constant                                        { $$ = $1; }
+;
+
+array_object_dereferencable:
+      fully_dereferencable                                  { $$ = $1; }
+    | constant                                              { $$ = $1; }
 ;
 
 callable_expr:
@@ -877,18 +909,23 @@ callable_expr:
 
 callable_variable:
       simple_variable                                       { $$ = Expr\Variable[$1]; }
-    | dereferencable '[' optional_expr ']'                  { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | constant '[' optional_expr ']'                        { $$ = Expr\ArrayDimFetch[$1, $3]; }
-    | dereferencable '{' expr '}'                           { $$ = Expr\ArrayDimFetch[$1, $3]; }
+    | array_object_dereferencable '[' optional_expr ']'     { $$ = Expr\ArrayDimFetch[$1, $3]; }
+    | array_object_dereferencable '{' expr '}'              { $$ = Expr\ArrayDimFetch[$1, $3]; }
     | function_call                                         { $$ = $1; }
-    | dereferencable T_OBJECT_OPERATOR property_name argument_list
+    | array_object_dereferencable T_OBJECT_OPERATOR property_name argument_list
           { $$ = Expr\MethodCall[$1, $3, $4]; }
 ;
 
+optional_plain_variable:
+      /* empty */                                           { $$ = null; }
+    | plain_variable                                        { $$ = $1; }
+;
+
 variable:
       callable_variable                                     { $$ = $1; }
     | static_member                                         { $$ = $1; }
-    | dereferencable T_OBJECT_OPERATOR property_name        { $$ = Expr\PropertyFetch[$1, $3]; }
+    | array_object_dereferencable T_OBJECT_OPERATOR property_name
+          { $$ = Expr\PropertyFetch[$1, $3]; }
 ;
 
 simple_variable:
@@ -933,22 +970,7 @@ property_name:
 ;
 
 list_expr:
-      T_LIST '(' list_expr_elements ')'                     { $$ = Expr\List_[$3]; }
-;
-
-list_expr_elements:
-      list_expr_elements ',' list_expr_element              { push($1, $3); }
-    | list_expr_element                                     { init($1); }
-;
-
-list_expr_element:
-      variable                                              { $$ = Expr\ArrayItem[$1, null, false]; }
-    | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
-    | list_expr                                             { $$ = Expr\ArrayItem[$1, null, false]; }
-    | expr T_DOUBLE_ARROW variable                          { $$ = Expr\ArrayItem[$3, $1, false]; }
-    | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1, true]; }
-    | expr T_DOUBLE_ARROW list_expr                         { $$ = Expr\ArrayItem[$3, $1, false]; }
-    | /* empty */                                           { $$ = null; }
+      T_LIST '(' inner_array_pair_list ')'                  { $$ = Expr\List_[$3]; }
 ;
 
 array_pair_list:
@@ -968,10 +990,13 @@ inner_array_pair_list:
 ;
 
 array_pair:
-      expr T_DOUBLE_ARROW expr                              { $$ = Expr\ArrayItem[$3, $1,   false]; }
-    | expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
-    | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
+      expr                                                  { $$ = Expr\ArrayItem[$1, null, false]; }
     | '&' variable                                          { $$ = Expr\ArrayItem[$2, null, true]; }
+    | list_expr                                             { $$ = Expr\ArrayItem[$1, null, false]; }
+    | expr T_DOUBLE_ARROW expr                              { $$ = Expr\ArrayItem[$3, $1,   false]; }
+    | expr T_DOUBLE_ARROW '&' variable                      { $$ = Expr\ArrayItem[$4, $1,   true]; }
+    | expr T_DOUBLE_ARROW list_expr                         { $$ = Expr\ArrayItem[$3, $1,   false]; }
+    | T_ELLIPSIS expr                                       { $$ = Expr\ArrayItem[$2, null, false, attributes(), true]; }
     | /* empty */                                           { $$ = null; }
 ;
 

grammar/rebuildParsers.php

@@ -13,10 +13,10 @@ $tmpResultFile  = __DIR__ . '/tmp_parser.php';
 $resultDir = __DIR__ . '/../lib/PhpParser/Parser';
 $tokensResultsFile = $resultDir . '/Tokens.php';
 
-// check for kmyacc.exe binary in this directory, otherwise fall back to global name
-$kmyacc = __DIR__ . '/kmyacc.exe';
-if (!file_exists($kmyacc)) {
-    $kmyacc = 'kmyacc';
+$kmyacc = getenv('KMYACC');
+if (!$kmyacc) {
+    // Use phpyacc from dev dependencies by default.
+    $kmyacc = PHP_BINARY . ' ' . __DIR__ . '/../vendor/bin/phpyacc';
 }
 
 $options = array_flip($argv);
@@ -59,8 +59,7 @@ foreach ($grammarFileToName as $grammarFile => $name) {
     $additionalArgs = $optionDebug ? '-t -v' : '';
 
     echo "Building $name parser.\n";
-    $output = trim(shell_exec("$kmyacc $additionalArgs -l -m $skeletonFile -p $name $tmpGrammarFile 2>&1"));
-    echo "Output: \"$output\"\n";
+    $output = execCmd("$kmyacc $additionalArgs -m $skeletonFile -p $name $tmpGrammarFile");
 
     $resultCode = file_get_contents($tmpResultFile);
     $resultCode = removeTrailingWhitespace($resultCode);
@@ -70,8 +69,7 @@ foreach ($grammarFileToName as $grammarFile => $name) {
     unlink($tmpResultFile);
 
     echo "Building token definition.\n";
-    $output = trim(shell_exec("$kmyacc -l -m $tokensTemplate $tmpGrammarFile 2>&1"));
-    assert($output === '');
+    $output = execCmd("$kmyacc -m $tokensTemplate $tmpGrammarFile");
     rename($tmpResultFile, $tokensResultsFile);
 
     if (!$optionKeepTmpGrammar) {
@@ -175,6 +173,15 @@ function resolveMacros($code) {
                 . ' else { ' . $args[0] . ' = null; }';
             }
 
+            if ('makeZeroLengthNop' == $name) {
+                assertArgs(2, $args, $name);
+
+                return '$startAttributes = ' . $args[1] . ';'
+                    . ' if (isset($startAttributes[\'comments\']))'
+                    . ' { ' . $args[0] . ' = new Stmt\Nop($this->createCommentNopAttributes($startAttributes[\'comments\'])); }'
+                    . ' else { ' . $args[0] . ' = null; }';
+            }
+
             if ('strKind' == $name) {
                 assertArgs(1, $args, $name);
 
@@ -222,6 +229,15 @@ function ensureDirExists($dir) {
     }
 }
 
+function execCmd($cmd) {
+    $output = trim(shell_exec("$cmd 2>&1"));
+    if ($output !== "") {
+        echo "> " . $cmd . "\n";
+        echo $output;
+    }
+    return $output;
+}
+
 //////////////////////////////
 /// Regex helper functions ///
 //////////////////////////////

grammar/tokens.y

@@ -10,7 +10,7 @@
 %right T_YIELD
 %right T_DOUBLE_ARROW
 %right T_YIELD_FROM
-%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL
+%left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL T_COALESCE_EQUAL
 %left '?' ':'
 %right T_COALESCE
 %left T_BOOLEAN_OR
@@ -64,6 +64,7 @@
 %token T_CONTINUE
 %token T_GOTO
 %token T_FUNCTION
+%token T_FN
 %token T_CONST
 %token T_RETURN
 %token T_TRY

lib/PhpParser/Builder/Param.php

@@ -12,7 +12,7 @@ class Param implements PhpParser\Builder
 
     protected $default = null;
 
-    /** @var string|Node\Name|Node\NullableType|null */
+    /** @var Node\Identifier|Node\Name|Node\NullableType|null */
     protected $type = null;
 
     protected $byRef = false;
@@ -44,7 +44,7 @@ class Param implements PhpParser\Builder
     /**
      * Sets type for the parameter.
      *
-     * @param string|Node\Name|Node\NullableType $type Parameter type
+     * @param string|Node\Name|Node\NullableType|Node\UnionType $type Parameter type
      *
      * @return $this The builder instance (for fluid interface)
      */
@@ -60,7 +60,7 @@ class Param implements PhpParser\Builder
     /**
      * Sets type for the parameter.
      *
-     * @param string|Node\Name|Node\NullableType $type Parameter type
+     * @param string|Node\Name|Node\NullableType|Node\UnionType $type Parameter type
      *
      * @return $this The builder instance (for fluid interface)
      *

lib/PhpParser/Builder/Property.php

@@ -4,6 +4,9 @@ namespace PhpParser\Builder;
 
 use PhpParser;
 use PhpParser\BuilderHelpers;
+use PhpParser\Node\Identifier;
+use PhpParser\Node\Name;
+use PhpParser\Node\NullableType;
 use PhpParser\Node\Stmt;
 
 class Property implements PhpParser\Builder
@@ -14,6 +17,9 @@ class Property implements PhpParser\Builder
     protected $default = null;
     protected $attributes = [];
 
+    /** @var null|Identifier|Name|NullableType */
+    protected $type;
+
     /**
      * Creates a property builder.
      *
@@ -95,6 +101,19 @@ class Property implements PhpParser\Builder
         return $this;
     }
 
+    /**
+     * Sets the property type for PHP 7.4+.
+     *
+     * @param string|Name|NullableType|Identifier $type
+     *
+     * @return $this
+     */
+    public function setType($type) {
+        $this->type = BuilderHelpers::normalizeType($type);
+
+        return $this;
+    }
+
     /**
      * Returns the built class node.
      *
@@ -106,7 +125,8 @@ class Property implements PhpParser\Builder
             [
                 new Stmt\PropertyProperty($this->name, $this->default)
             ],
-            $this->attributes
+            $this->attributes,
+            $this->type
         );
     }
 }

lib/PhpParser/BuilderFactory.php

@@ -8,7 +8,6 @@ use PhpParser\Node\Expr\BinaryOp\Concat;
 use PhpParser\Node\Identifier;
 use PhpParser\Node\Name;
 use PhpParser\Node\Scalar\String_;
-use PhpParser\Node\Stmt;
 use PhpParser\Node\Stmt\Use_;
 
 class BuilderFactory
@@ -77,7 +76,7 @@ class BuilderFactory
      * @return Builder\TraitUseAdaptation The create trait use adaptation builder
      */
     public function traitUseAdaptation($trait, $method = null) : Builder\TraitUseAdaptation {
-        if (is_null($method)) {
+        if ($method === null) {
             $method = $trait;
             $trait = null;
         }

lib/PhpParser/BuilderHelpers.php

@@ -8,6 +8,7 @@ use PhpParser\Node\Name;
 use PhpParser\Node\NullableType;
 use PhpParser\Node\Scalar;
 use PhpParser\Node\Stmt;
+use PhpParser\Node\UnionType;
 
 /**
  * This class defines helpers used in the implementation of builders. Don't use it directly.
@@ -158,16 +159,19 @@ final class BuilderHelpers
      * In particular, builtin types become Identifiers, custom types become Names and nullables
      * are wrapped in NullableType nodes.
      *
-     * @param string|Name|Identifier|NullableType $type The type to normalize
+     * @param string|Name|Identifier|NullableType|UnionType $type The type to normalize
      *
-     * @return Name|Identifier|NullableType The normalized type
+     * @return Name|Identifier|NullableType|UnionType The normalized type
      */
     public static function normalizeType($type) {
         if (!is_string($type)) {
-            if (!$type instanceof Name && !$type instanceof Identifier
-                    && !$type instanceof NullableType) {
+            if (
+                !$type instanceof Name && !$type instanceof Identifier &&
+                !$type instanceof NullableType && !$type instanceof UnionType
+            ) {
                 throw new \LogicException(
-                    'Type must be a string, or an instance of Name, Identifier or NullableType');
+                    'Type must be a string, or an instance of Name, Identifier, NullableType or UnionType'
+                );
             }
             return $type;
         }
@@ -179,7 +183,7 @@ final class BuilderHelpers
         }
 
         $builtinTypes = [
-            'array', 'callable', 'string', 'int', 'float', 'bool', 'iterable', 'void', 'object'
+            'array', 'callable', 'string', 'int', 'float', 'bool', 'iterable', 'void', 'object', 'mixed'
         ];
 
         $lowerType = strtolower($type);
@@ -193,7 +197,11 @@ final class BuilderHelpers
             throw new \LogicException('void type cannot be nullable');
         }
 
-        return $nullable ? new Node\NullableType($type) : $type;
+        if ($nullable && (string) $type === 'mixed') {
+            throw new \LogicException('mixed type cannot be nullable');
+        }
+
+        return $nullable ? new NullableType($type) : $type;
     }
 
     /**

lib/PhpParser/Comment.php

@@ -5,9 +5,12 @@ namespace PhpParser;
 class Comment implements \JsonSerializable
 {
     protected $text;
-    protected $line;
-    protected $filePos;
-    protected $tokenPos;
+    protected $startLine;
+    protected $startFilePos;
+    protected $startTokenPos;
+    protected $endLine;
+    protected $endFilePos;
+    protected $endTokenPos;
 
     /**
      * Constructs a comment node.
@@ -18,12 +21,17 @@ class Comment implements \JsonSerializable
      * @param int    $startTokenPos Token offset the comment started on
      */
     public function __construct(
-        string $text, int $startLine = -1, int $startFilePos = -1, int $startTokenPos = -1
+        string $text,
+        int $startLine = -1, int $startFilePos = -1, int $startTokenPos = -1,
+        int $endLine = -1, int $endFilePos = -1, int $endTokenPos = -1
     ) {
         $this->text = $text;
-        $this->line = $startLine;
-        $this->filePos = $startFilePos;
-        $this->tokenPos = $startTokenPos;
+        $this->startLine = $startLine;
+        $this->startFilePos = $startFilePos;
+        $this->startTokenPos = $startTokenPos;
+        $this->endLine = $endLine;
+        $this->endFilePos = $endFilePos;
+        $this->endTokenPos = $endTokenPos;
     }
 
     /**
@@ -38,28 +46,88 @@ class Comment implements \JsonSerializable
     /**
      * Gets the line number the comment started on.
      *
-     * @return int Line number
+     * @return int Line number (or -1 if not available)
      */
-    public function getLine() : int {
-        return $this->line;
+    public function getStartLine() : int {
+        return $this->startLine;
     }
 
     /**
      * Gets the file offset the comment started on.
      *
-     * @return int File offset
+     * @return int File offset (or -1 if not available)
      */
-    public function getFilePos() : int {
-        return $this->filePos;
+    public function getStartFilePos() : int {
+        return $this->startFilePos;
     }
 
     /**
      * Gets the token offset the comment started on.
      *
+     * @return int Token offset (or -1 if not available)
+     */
+    public function getStartTokenPos() : int {
+        return $this->startTokenPos;
+    }
+
+    /**
+     * Gets the line number the comment ends on.
+     *
+     * @return int Line number (or -1 if not available)
+     */
+    public function getEndLine() : int {
+        return $this->endLine;
+    }
+
+    /**
+     * Gets the file offset the comment ends on.
+     *
+     * @return int File offset (or -1 if not available)
+     */
+    public function getEndFilePos() : int {
+        return $this->endFilePos;
+    }
+
+    /**
+     * Gets the token offset the comment ends on.
+     *
+     * @return int Token offset (or -1 if not available)
+     */
+    public function getEndTokenPos() : int {
+        return $this->endTokenPos;
+    }
+
+    /**
+     * Gets the line number the comment started on.
+     *
+     * @deprecated Use getStartLine() instead
+     *
+     * @return int Line number
+     */
+    public function getLine() : int {
+        return $this->startLine;
+    }
+
+    /**
+     * Gets the file offset the comment started on.
+     *
+     * @deprecated Use getStartFilePos() instead
+     *
+     * @return int File offset
+     */
+    public function getFilePos() : int {
+        return $this->startFilePos;
+    }
+
+    /**
+     * Gets the token offset the comment started on.
+     *
+     * @deprecated Use getStartTokenPos() instead
+     *
      * @return int Token offset
      */
     public function getTokenPos() : int {
-        return $this->tokenPos;
+        return $this->startTokenPos;
     }
 
     /**
@@ -159,9 +227,13 @@ class Comment implements \JsonSerializable
         return [
             'nodeType' => $type,
             'text' => $this->text,
-            'line' => $this->line,
-            'filePos' => $this->filePos,
-            'tokenPos' => $this->tokenPos,
+            // TODO: Rename these to include "start".
+            'line' => $this->startLine,
+            'filePos' => $this->startFilePos,
+            'tokenPos' => $this->startTokenPos,
+            'endLine' => $this->endLine,
+            'endFilePos' => $this->endFilePos,
+            'endTokenPos' => $this->endTokenPos,
         ];
     }
 }

lib/PhpParser/Internal/PrintableNewAnonClassNode.php

@@ -41,7 +41,6 @@ class PrintableNewAnonClassNode extends Expr
         assert($class instanceof Node\Stmt\Class_);
         // We don't assert that $class->name is null here, to allow consumers to assign unique names
         // to anonymous classes for their own purposes. We simplify ignore the name here.
-        assert($class->name === null);
         return new self(
             $newNode->args, $class->extends, $class->implements,
             $class->stmts, $newNode->getAttributes()

lib/PhpParser/JsonDecoder.php

@@ -73,7 +73,9 @@ class JsonDecoder
         }
 
         return new $className(
-            $value['text'], $value['line'] ?? -1, $value['filePos'] ?? -1, $value['tokenPos'] ?? -1
+            $value['text'],
+            $value['line'] ?? -1, $value['filePos'] ?? -1, $value['tokenPos'] ?? -1,
+            $value['endLine'] ?? -1, $value['endFilePos'] ?? -1, $value['endTokenPos'] ?? -1
         );
     }
 

lib/PhpParser/Lexer.php

@@ -16,7 +16,13 @@ class Lexer
     protected $tokenMap;
     protected $dropTokens;
 
-    protected $usedAttributes;
+    private $attributeStartLineUsed;
+    private $attributeEndLineUsed;
+    private $attributeStartTokenPosUsed;
+    private $attributeEndTokenPosUsed;
+    private $attributeStartFilePosUsed;
+    private $attributeEndFilePosUsed;
+    private $attributeCommentsUsed;
 
     /**
      * Creates a Lexer.
@@ -31,18 +37,28 @@ class Lexer
         // map from internal tokens to PhpParser tokens
         $this->tokenMap = $this->createTokenMap();
 
+        // Compatibility define for PHP < 7.4
+        if (!defined('T_BAD_CHARACTER')) {
+            \define('T_BAD_CHARACTER', -1);
+        }
+
         // map of tokens to drop while lexing (the map is only used for isset lookup,
         // that's why the value is simply set to 1; the value is never actually used.)
         $this->dropTokens = array_fill_keys(
-            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT], 1
+            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
         );
 
-        // the usedAttributes member is a map of the used attribute names to a dummy
-        // value (here "true")
-        $options += [
-            'usedAttributes' => ['comments', 'startLine', 'endLine'],
-        ];
-        $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
+        $defaultAttributes = ['comments', 'startLine', 'endLine'];
+        $usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
+
+        // Create individual boolean properties to make these checks faster.
+        $this->attributeStartLineUsed = isset($usedAttributes['startLine']);
+        $this->attributeEndLineUsed = isset($usedAttributes['endLine']);
+        $this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
+        $this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
+        $this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
+        $this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
+        $this->attributeCommentsUsed = isset($usedAttributes['comments']);
     }
 
     /**
@@ -81,13 +97,9 @@ class Lexer
     }
 
     private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
+        $tokens = [];
         for ($i = $start; $i < $end; $i++) {
             $chr = $this->code[$i];
-            if ($chr === 'b' || $chr === 'B') {
-                // HHVM does not treat b" tokens correctly, so ignore these
-                continue;
-            }
-
             if ($chr === "\0") {
                 // PHP cuts error message after null byte, so need special case
                 $errorMsg = 'Unexpected null byte';
@@ -97,6 +109,7 @@ class Lexer
                 );
             }
 
+            $tokens[] = [\T_BAD_CHARACTER, $chr, $line];
             $errorHandler->handleError(new Error($errorMsg, [
                 'startLine' => $line,
                 'endLine' => $line,
@@ -104,6 +117,7 @@ class Lexer
                 'endFilePos' => $i,
             ]));
         }
+        return $tokens;
     }
 
     /**
@@ -129,6 +143,16 @@ class Lexer
             return true;
         }
 
+        if (PHP_VERSION_ID >= 80000) {
+            // PHP 8 converts the "bad character" case into a parse error, rather than treating
+            // it as a lexing warning. To preserve previous behavior, we need to assume that an
+            // error occurred.
+            // TODO: We should handle this the same way as PHP 8: Only generate T_BAD_CHARACTER
+            // token here (for older PHP versions) and leave generationg of the actual parse error
+            // to the parser. This will also save the full token scan on PHP 8 here.
+            return true;
+        }
+
         return null !== error_get_last();
     }
 
@@ -144,16 +168,29 @@ class Lexer
 
         $filePos = 0;
         $line = 1;
-        foreach ($this->tokens as $token) {
+        $numTokens = \count($this->tokens);
+        for ($i = 0; $i < $numTokens; $i++) {
+            $token = $this->tokens[$i];
+
+            // Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
+            // In this case we only need to emit an error.
+            if ($token[0] === \T_BAD_CHARACTER) {
+                $this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
+            }
+
             $tokenValue = \is_string($token) ? $token : $token[1];
             $tokenLen = \strlen($tokenValue);
 
             if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
                 // Something is missing, must be an invalid character
                 $nextFilePos = strpos($this->code, $tokenValue, $filePos);
-                $this->handleInvalidCharacterRange(
+                $badCharTokens = $this->handleInvalidCharacterRange(
                     $filePos, $nextFilePos, $line, $errorHandler);
                 $filePos = (int) $nextFilePos;
+
+                array_splice($this->tokens, $i, 0, $badCharTokens);
+                $numTokens += \count($badCharTokens);
+                $i += \count($badCharTokens);
             }
 
             $filePos += $tokenLen;
@@ -176,8 +213,9 @@ class Lexer
                 $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
             } else {
                 // Invalid characters at the end of the input
-                $this->handleInvalidCharacterRange(
+                $badCharTokens = $this->handleInvalidCharacterRange(
                     $filePos, \strlen($this->code), $line, $errorHandler);
+                $this->tokens = array_merge($this->tokens, $badCharTokens);
             }
             return;
         }
@@ -230,13 +268,13 @@ class Lexer
                 $token = "\0";
             }
 
-            if (isset($this->usedAttributes['startLine'])) {
+            if ($this->attributeStartLineUsed) {
                 $startAttributes['startLine'] = $this->line;
             }
-            if (isset($this->usedAttributes['startTokenPos'])) {
+            if ($this->attributeStartTokenPosUsed) {
                 $startAttributes['startTokenPos'] = $this->pos;
             }
-            if (isset($this->usedAttributes['startFilePos'])) {
+            if ($this->attributeStartFilePosUsed) {
                 $startAttributes['startFilePos'] = $this->filePos;
             }
 
@@ -262,27 +300,33 @@ class Lexer
                 $this->line += substr_count($value, "\n");
                 $this->filePos += \strlen($value);
             } else {
+                $origLine = $this->line;
+                $origFilePos = $this->filePos;
+                $this->line += substr_count($token[1], "\n");
+                $this->filePos += \strlen($token[1]);
+
                 if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
-                    if (isset($this->usedAttributes['comments'])) {
+                    if ($this->attributeCommentsUsed) {
                         $comment = \T_DOC_COMMENT === $token[0]
-                            ? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
-                            : new Comment($token[1], $this->line, $this->filePos, $this->pos);
+                            ? new Comment\Doc($token[1],
+                                $origLine, $origFilePos, $this->pos,
+                                $this->line, $this->filePos - 1, $this->pos)
+                            : new Comment($token[1],
+                                $origLine, $origFilePos, $this->pos,
+                                $this->line, $this->filePos - 1, $this->pos);
                         $startAttributes['comments'][] = $comment;
                     }
                 }
-
-                $this->line += substr_count($token[1], "\n");
-                $this->filePos += \strlen($token[1]);
                 continue;
             }
 
-            if (isset($this->usedAttributes['endLine'])) {
+            if ($this->attributeEndLineUsed) {
                 $endAttributes['endLine'] = $this->line;
             }
-            if (isset($this->usedAttributes['endTokenPos'])) {
+            if ($this->attributeEndTokenPosUsed) {
                 $endAttributes['endTokenPos'] = $this->pos;
             }
-            if (isset($this->usedAttributes['endFilePos'])) {
+            if ($this->attributeEndFilePosUsed) {
                 $endAttributes['endFilePos'] = $this->filePos - 1;
             }
 

lib/PhpParser/Lexer/Emulative.php

@@ -4,26 +4,61 @@ namespace PhpParser\Lexer;
 
 use PhpParser\Error;
 use PhpParser\ErrorHandler;
+use PhpParser\Lexer;
+use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
+use PhpParser\Lexer\TokenEmulator\TokenEmulatorInterface;
+use PhpParser\Parser\Tokens;
 
-class Emulative extends \PhpParser\Lexer
+class Emulative extends Lexer
 {
     const PHP_7_3 = '7.3.0dev';
+    const PHP_7_4 = '7.4.0dev';
+
+    const T_COALESCE_EQUAL = 1007;
+    const T_FN = 1008;
+
+    const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
+/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
+(?:.*\r?\n)*?
+(?<indentation>\h*)\2(?![a-zA-Z0-9_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
+REGEX;
+
+    /** @var mixed[] Patches used to reverse changes introduced in the code */
+    private $patches = [];
+
+    /** @var TokenEmulatorInterface[] */
+    private $tokenEmulators = [];
 
     /**
-     * @var array Patches used to reverse changes introduced in the code
+     * @param mixed[] $options
      */
-    private $patches;
+    public function __construct(array $options = [])
+    {
+        parent::__construct($options);
+
+        $this->tokenEmulators[] = new FnTokenEmulator();
+        $this->tokenEmulators[] = new CoaleseEqualTokenEmulator();
+        $this->tokenEmulators[] = new NumericLiteralSeparatorEmulator();
+
+        $this->tokenMap[self::T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
+        $this->tokenMap[self::T_FN] = Tokens::T_FN;
+    }
 
     public function startLexing(string $code, ErrorHandler $errorHandler = null) {
         $this->patches = [];
-        $preparedCode = $this->prepareCode($code);
-        if (null === $preparedCode) {
+
+        if ($this->isEmulationNeeded($code) === false) {
             // Nothing to emulate, yay
             parent::startLexing($code, $errorHandler);
             return;
         }
 
         $collector = new ErrorHandler\Collecting();
+
+        // 1. emulation of heredoc and nowdoc new syntax
+        $preparedCode = $this->processHeredocNowdoc($code);
         parent::startLexing($preparedCode, $collector);
         $this->fixupTokens();
 
@@ -34,32 +69,34 @@ class Emulative extends \PhpParser\Lexer
                 $errorHandler->handleError($error);
             }
         }
+
+        // add token emulation
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                $this->tokens = $emulativeToken->emulate($code, $this->tokens);
+            }
+        }
     }
 
-    /**
-     * Prepares code for emulation. If nothing has to be emulated null is returned.
-     *
-     * @param string $code
-     * @return null|string
-     */
-    private function prepareCode(string $code) {
+    private function isHeredocNowdocEmulationNeeded(string $code): bool
+    {
+        // skip version where this works without emulation
         if (version_compare(\PHP_VERSION, self::PHP_7_3, '>=')) {
-            return null;
+            return false;
         }
 
-        if (strpos($code, '<<<') === false) {
-            // Definitely doesn't contain heredoc/nowdoc
-            return null;
+        return strpos($code, '<<<') !== false;
+    }
+
+    private function processHeredocNowdoc(string $code): string
+    {
+        if ($this->isHeredocNowdocEmulationNeeded($code) === false) {
+            return $code;
         }
 
-        $flexibleDocStringRegex = <<<'REGEX'
-/<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
-(?:.*\r?\n)*?
-(?<indentation>\h*)\2(?![a-zA-Z_\x80-\xff])(?<separator>(?:;?[\r\n])?)/x
-REGEX;
-        if (!preg_match_all($flexibleDocStringRegex, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
+        if (!preg_match_all(self::FLEXIBLE_DOC_STRING_REGEX, $code, $matches, PREG_SET_ORDER|PREG_OFFSET_CAPTURE)) {
             // No heredoc/nowdoc found
-            return null;
+            return $code;
         }
 
         // Keep track of how much we need to adjust string offsets due to the modifications we
@@ -93,19 +130,29 @@ REGEX;
             }
         }
 
-        if (empty($this->patches)) {
-            // We did not end up emulating anything
-            return null;
-        }
-
         return $code;
     }
 
-    private function fixupTokens() {
-        assert(count($this->patches) > 0);
+    private function isEmulationNeeded(string $code): bool
+    {
+        foreach ($this->tokenEmulators as $emulativeToken) {
+            if ($emulativeToken->isEmulationNeeded($code)) {
+                return true;
+            }
+        }
+
+        return $this->isHeredocNowdocEmulationNeeded($code);
+    }
+
+    private function fixupTokens()
+    {
+        if (\count($this->patches) === 0) {
+            return;
+        }
 
         // Load first patch
         $patchIdx = 0;
+
         list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
 
         // We use a manual loop over the tokens, because we modify the array on the fly
@@ -200,4 +247,4 @@ REGEX;
             $error->setAttributes($attrs);
         }
     }
-}
+}

lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php

@@ -0,0 +1,41 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return strpos($code, '??=') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $line = 1;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            if (isset($tokens[$i + 1])) {
+                if ($tokens[$i][0] === T_COALESCE && $tokens[$i + 1] === '=') {
+                    array_splice($tokens, $i, 2, [
+                        [Emulative::T_COALESCE_EQUAL, '??=', $line]
+                    ]);
+                    $c--;
+                    continue;
+                }
+            }
+            if (\is_array($tokens[$i])) {
+                $line += substr_count($tokens[$i][1], "\n");
+            }
+        }
+
+        return $tokens;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php

@@ -0,0 +1,53 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class FnTokenEmulator implements TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return strpos($code, 'fn') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        foreach ($tokens as $i => $token) {
+            if ($token[0] === T_STRING && $token[1] === 'fn') {
+                $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $i);
+                if ($previousNonSpaceToken !== null && $previousNonSpaceToken[0] === T_OBJECT_OPERATOR) {
+                    continue;
+                }
+
+                $tokens[$i][0] = Emulative::T_FN;
+            }
+        }
+
+        return $tokens;
+    }
+
+    /**
+     * @param mixed[] $tokens
+     * @return mixed[]|null
+     */
+    private function getPreviousNonSpaceToken(array $tokens, int $start)
+    {
+        for ($i = $start - 1; $i >= 0; --$i) {
+            if ($tokens[$i][0] === T_WHITESPACE) {
+                continue;
+            }
+
+            return $tokens[$i];
+        }
+
+        return null;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php

@@ -0,0 +1,99 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Lexer\Emulative;
+
+final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
+{
+    const BIN = '(?:0b[01]+(?:_[01]+)*)';
+    const HEX = '(?:0x[0-9a-f]+(?:_[0-9a-f]+)*)';
+    const DEC = '(?:[0-9]+(?:_[0-9]+)*)';
+    const SIMPLE_FLOAT = '(?:' . self::DEC . '\.' . self::DEC . '?|\.' . self::DEC . ')';
+    const EXP = '(?:e[+-]?' . self::DEC . ')';
+    const FLOAT = '(?:' . self::SIMPLE_FLOAT . self::EXP . '?|' . self::DEC . self::EXP . ')';
+    const NUMBER = '~' . self::FLOAT . '|' . self::BIN . '|' . self::HEX . '|' . self::DEC . '~iA';
+
+    public function isEmulationNeeded(string $code) : bool
+    {
+        // skip version where this is supported
+        if (version_compare(\PHP_VERSION, Emulative::PHP_7_4, '>=')) {
+            return false;
+        }
+
+        return preg_match('~[0-9]_[0-9]~', $code)
+            || preg_match('~0x[0-9a-f]+_[0-9a-f]~i', $code);
+    }
+
+    public function emulate(string $code, array $tokens): array
+    {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        $codeOffset = 0;
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            $tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
+
+            if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
+            assert($res, "No number at number token position");
+
+            $match = $matches[0];
+            $matchLen = \strlen($match);
+            if ($matchLen === $tokenLen) {
+                // Original token already holds the full number.
+                $codeOffset += $tokenLen;
+                continue;
+            }
+
+            $tokenKind = $this->resolveIntegerOrFloatToken($match);
+            $newTokens = [[$tokenKind, $match, $token[2]]];
+
+            $numTokens = 1;
+            $len = $tokenLen;
+            while ($matchLen > $len) {
+                $nextToken = $tokens[$i + $numTokens];
+                $nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
+                $nextTokenLen = \strlen($nextTokenText);
+
+                $numTokens++;
+                if ($matchLen < $len + $nextTokenLen) {
+                    // Split trailing characters into a partial token.
+                    assert(is_array($nextToken), "Partial token should be an array token");
+                    $partialText = substr($nextTokenText, $matchLen - $len);
+                    $newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
+                    break;
+                }
+
+                $len += $nextTokenLen;
+            }
+
+            array_splice($tokens, $i, $numTokens, $newTokens);
+            $c -= $numTokens - \count($newTokens);
+            $codeOffset += $matchLen;
+        }
+
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int
+    {
+        $str = str_replace('_', '', $str);
+
+        if (stripos($str, '0b') === 0) {
+            $num = bindec($str);
+        } elseif (stripos($str, '0x') === 0) {
+            $num = hexdec($str);
+        } elseif (stripos($str, '0') === 0 && ctype_digit($str)) {
+            $num = octdec($str);
+        } else {
+            $num = +$str;
+        }
+
+        return is_float($num) ? T_DNUMBER : T_LNUMBER;
+    }
+}

lib/PhpParser/Lexer/TokenEmulator/TokenEmulatorInterface.php

@@ -0,0 +1,14 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+/** @internal */
+interface TokenEmulatorInterface
+{
+    public function isEmulationNeeded(string $code): bool;
+
+    /**
+     * @return array Modified Tokens
+     */
+    public function emulate(string $code, array $tokens): array;
+}

lib/PhpParser/Node.php

@@ -95,8 +95,6 @@ interface Node
     /**
      * Gets the doc comment of the node.
      *
-     * The doc comment has to be the last comment associated with the node.
-     *
      * @return null|Comment\Doc Doc comment object or null
      */
     public function getDocComment();

lib/PhpParser/Node/Arg.php

@@ -22,7 +22,7 @@ class Arg extends NodeAbstract
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $value, bool $byRef = false, bool $unpack = false, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
         $this->byRef = $byRef;
         $this->unpack = $unpack;

lib/PhpParser/Node/Const_.php

@@ -22,7 +22,7 @@ class Const_ extends NodeAbstract
      * @param array             $attributes Additional attributes
      */
     public function __construct($name, Expr $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
         $this->value = $value;
     }

lib/PhpParser/Node/Expr/ArrayDimFetch.php

@@ -19,7 +19,7 @@ class ArrayDimFetch extends Expr
      * @param array     $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $dim = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->dim = $dim;
     }

lib/PhpParser/Node/Expr/ArrayItem.php

@@ -12,6 +12,8 @@ class ArrayItem extends Expr
     public $value;
     /** @var bool Whether to assign by reference */
     public $byRef;
+    /** @var bool Whether to unpack the argument */
+    public $unpack;
 
     /**
      * Constructs an array item node.
@@ -21,17 +23,18 @@ class ArrayItem extends Expr
      * @param bool      $byRef      Whether to assign by reference
      * @param array     $attributes Additional attributes
      */
-    public function __construct(Expr $value, Expr $key = null, bool $byRef = false, array $attributes = []) {
-        parent::__construct($attributes);
+    public function __construct(Expr $value, Expr $key = null, bool $byRef = false, array $attributes = [], bool $unpack = false) {
+        $this->attributes = $attributes;
         $this->key = $key;
         $this->value = $value;
         $this->byRef = $byRef;
+        $this->unpack = $unpack;
     }
 
     public function getSubNodeNames() : array {
-        return ['key', 'value', 'byRef'];
+        return ['key', 'value', 'byRef', 'unpack'];
     }
-    
+
     public function getType() : string {
         return 'Expr_ArrayItem';
     }

lib/PhpParser/Node/Expr/Array_.php

@@ -20,7 +20,7 @@ class Array_ extends Expr
      * @param array       $attributes Additional attributes
      */
     public function __construct(array $items = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->items = $items;
     }
 

lib/PhpParser/Node/Expr/ArrowFunction.php

@@ -0,0 +1,71 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node\Expr;
+
+use PhpParser\Node;
+use PhpParser\Node\Expr;
+use PhpParser\Node\FunctionLike;
+
+class ArrowFunction extends Expr implements FunctionLike
+{
+    /** @var bool */
+    public $static;
+
+    /** @var bool */
+    public $byRef;
+
+    /** @var Node\Param[] */
+    public $params = [];
+
+    /** @var null|Node\Identifier|Node\Name|Node\NullableType|Node\UnionType */
+    public $returnType;
+
+    /** @var Expr */
+    public $expr;
+
+    /**
+     * @param array $subNodes   Array of the following optional subnodes:
+     *                          'static'     => false   : Whether the closure is static
+     *                          'byRef'      => false   : Whether to return by reference
+     *                          'params'     => array() : Parameters
+     *                          'returnType' => null    : Return type
+     *                          'expr'       => Expr    : Expression body
+     * @param array $attributes Additional attributes
+     */
+    public function __construct(array $subNodes = [], array $attributes = []) {
+        $this->attributes = $attributes;
+        $this->static = $subNodes['static'] ?? false;
+        $this->byRef = $subNodes['byRef'] ?? false;
+        $this->params = $subNodes['params'] ?? [];
+        $returnType = $subNodes['returnType'] ?? null;
+        $this->returnType = \is_string($returnType) ? new Node\Identifier($returnType) : $returnType;
+        $this->expr = $subNodes['expr'] ?? null;
+    }
+
+    public function getSubNodeNames() : array {
+        return ['static', 'byRef', 'params', 'returnType', 'expr'];
+    }
+
+    public function returnsByRef() : bool {
+        return $this->byRef;
+    }
+
+    public function getParams() : array {
+        return $this->params;
+    }
+
+    public function getReturnType() {
+        return $this->returnType;
+    }
+
+    /**
+     * @return Node\Stmt\Return_[]
+     */
+    public function getStmts() : array {
+        return [new Node\Stmt\Return_($this->expr)];
+    }
+
+    public function getType() : string {
+        return 'Expr_ArrowFunction';
+    }
+}

lib/PhpParser/Node/Expr/Assign.php

@@ -19,7 +19,7 @@ class Assign extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }

lib/PhpParser/Node/Expr/AssignOp.php

@@ -19,7 +19,7 @@ abstract class AssignOp extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }

lib/PhpParser/Node/Expr/AssignOp/Coalesce.php

@@ -0,0 +1,12 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Node\Expr\AssignOp;
+
+use PhpParser\Node\Expr\AssignOp;
+
+class Coalesce extends AssignOp
+{
+    public function getType() : string {
+        return 'Expr_AssignOp_Coalesce';
+    }
+}

lib/PhpParser/Node/Expr/AssignRef.php

@@ -19,7 +19,7 @@ class AssignRef extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->expr = $expr;
     }

lib/PhpParser/Node/Expr/BinaryOp.php

@@ -19,7 +19,7 @@ abstract class BinaryOp extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $left, Expr $right, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->left = $left;
         $this->right = $right;
     }

lib/PhpParser/Node/Expr/BitwiseNot.php

@@ -16,7 +16,7 @@ class BitwiseNot extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/BooleanNot.php

@@ -16,7 +16,7 @@ class BooleanNot extends Expr
      * @param array               $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Cast.php

@@ -16,7 +16,7 @@ abstract class Cast extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Cast/Double.php

@@ -6,6 +6,11 @@ use PhpParser\Node\Expr\Cast;
 
 class Double extends Cast
 {
+    // For use in "kind" attribute
+    const KIND_DOUBLE = 1; // "double" syntax
+    const KIND_FLOAT = 2;  // "float" syntax
+    const KIND_REAL = 3; // "real" syntax
+
     public function getType() : string {
         return 'Expr_Cast_Double';
     }

lib/PhpParser/Node/Expr/ClassConstFetch.php

@@ -21,7 +21,7 @@ class ClassConstFetch extends Expr
      * @param array                   $attributes Additional attributes
      */
     public function __construct($class, $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->class = $class;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
     }

lib/PhpParser/Node/Expr/Clone_.php

@@ -16,7 +16,7 @@ class Clone_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Closure.php

@@ -16,7 +16,7 @@ class Closure extends Expr implements FunctionLike
     public $params;
     /** @var ClosureUse[] use()s */
     public $uses;
-    /** @var null|Node\Identifier|Node\Name|Node\NullableType Return type */
+    /** @var null|Node\Identifier|Node\Name|Node\NullableType|Node\UnionType Return type */
     public $returnType;
     /** @var Node\Stmt[] Statements */
     public $stmts;
@@ -34,7 +34,7 @@ class Closure extends Expr implements FunctionLike
      * @param array $attributes Additional attributes
      */
     public function __construct(array $subNodes = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->static = $subNodes['static'] ?? false;
         $this->byRef = $subNodes['byRef'] ?? false;
         $this->params = $subNodes['params'] ?? [];

lib/PhpParser/Node/Expr/ClosureUse.php

@@ -19,7 +19,7 @@ class ClosureUse extends Expr
      * @param array         $attributes Additional attributes
      */
     public function __construct(Expr\Variable $var, bool $byRef = false, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->byRef = $byRef;
     }

lib/PhpParser/Node/Expr/ConstFetch.php

@@ -17,7 +17,7 @@ class ConstFetch extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Name $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = $name;
     }
 

lib/PhpParser/Node/Expr/Empty_.php

@@ -16,7 +16,7 @@ class Empty_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Error.php

@@ -18,7 +18,7 @@ class Error extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
     }
 
     public function getSubNodeNames() : array {

lib/PhpParser/Node/Expr/ErrorSuppress.php

@@ -16,7 +16,7 @@ class ErrorSuppress extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Eval_.php

@@ -16,7 +16,7 @@ class Eval_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Exit_.php

@@ -20,7 +20,7 @@ class Exit_ extends Expr
      * @param array                    $attributes Additional attributes
      */
     public function __construct(Expr $expr = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/FuncCall.php

@@ -20,7 +20,7 @@ class FuncCall extends Expr
      * @param array          $attributes Additional attributes
      */
     public function __construct($name, array $args = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = $name;
         $this->args = $args;
     }

lib/PhpParser/Node/Expr/Include_.php

@@ -24,7 +24,7 @@ class Include_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, int $type, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
         $this->type = $type;
     }

lib/PhpParser/Node/Expr/Instanceof_.php

@@ -20,7 +20,7 @@ class Instanceof_ extends Expr
      * @param array     $attributes Additional attributes
      */
     public function __construct(Expr $expr, $class, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
         $this->class = $class;
     }

lib/PhpParser/Node/Expr/Isset_.php

@@ -16,7 +16,7 @@ class Isset_ extends Expr
      * @param array  $attributes Additional attributes
      */
     public function __construct(array $vars, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->vars = $vars;
     }
 

lib/PhpParser/Node/Expr/List_.php

@@ -16,7 +16,7 @@ class List_ extends Expr
      * @param array              $attributes Additional attributes
      */
     public function __construct(array $items, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->items = $items;
     }
 

lib/PhpParser/Node/Expr/MethodCall.php

@@ -24,7 +24,7 @@ class MethodCall extends Expr
      * @param array                  $attributes Additional attributes
      */
     public function __construct(Expr $var, $name, array $args = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
         $this->args = $args;

lib/PhpParser/Node/Expr/New_.php

@@ -20,7 +20,7 @@ class New_ extends Expr
      * @param array                           $attributes Additional attributes
      */
     public function __construct($class, array $args = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->class = $class;
         $this->args = $args;
     }

lib/PhpParser/Node/Expr/PostDec.php

@@ -16,7 +16,7 @@ class PostDec extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
     }
 

lib/PhpParser/Node/Expr/PostInc.php

@@ -16,7 +16,7 @@ class PostInc extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
     }
 

lib/PhpParser/Node/Expr/PreDec.php

@@ -16,7 +16,7 @@ class PreDec extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
     }
 

lib/PhpParser/Node/Expr/PreInc.php

@@ -16,7 +16,7 @@ class PreInc extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $var, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
     }
 

lib/PhpParser/Node/Expr/Print_.php

@@ -16,7 +16,7 @@ class Print_ extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/PropertyFetch.php

@@ -20,7 +20,7 @@ class PropertyFetch extends Expr
      * @param array                  $attributes Additional attributes
      */
     public function __construct(Expr $var, $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->var = $var;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
     }

lib/PhpParser/Node/Expr/ShellExec.php

@@ -16,7 +16,7 @@ class ShellExec extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(array $parts, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->parts = $parts;
     }
 

lib/PhpParser/Node/Expr/StaticCall.php

@@ -24,7 +24,7 @@ class StaticCall extends Expr
      * @param array                  $attributes Additional attributes
      */
     public function __construct($class, $name, array $args = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->class = $class;
         $this->name = \is_string($name) ? new Identifier($name) : $name;
         $this->args = $args;

lib/PhpParser/Node/Expr/StaticPropertyFetch.php

@@ -21,7 +21,7 @@ class StaticPropertyFetch extends Expr
      * @param array                         $attributes Additional attributes
      */
     public function __construct($class, $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->class = $class;
         $this->name = \is_string($name) ? new VarLikeIdentifier($name) : $name;
     }

lib/PhpParser/Node/Expr/Ternary.php

@@ -22,7 +22,7 @@ class Ternary extends Expr
      * @param array                    $attributes Additional attributes
      */
     public function __construct(Expr $cond, $if, Expr $else, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->cond = $cond;
         $this->if = $if;
         $this->else = $else;

lib/PhpParser/Node/Expr/UnaryMinus.php

@@ -16,7 +16,7 @@ class UnaryMinus extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/UnaryPlus.php

@@ -16,7 +16,7 @@ class UnaryPlus extends Expr
      * @param array               $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Variable.php

@@ -16,7 +16,7 @@ class Variable extends Expr
      * @param array                      $attributes Additional attributes
      */
     public function __construct($name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = $name;
     }
 

lib/PhpParser/Node/Expr/YieldFrom.php

@@ -16,7 +16,7 @@ class YieldFrom extends Expr
      * @param array $attributes Additional attributes
      */
     public function __construct(Expr $expr, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->expr = $expr;
     }
 

lib/PhpParser/Node/Expr/Yield_.php

@@ -19,7 +19,7 @@ class Yield_ extends Expr
      * @param array     $attributes Additional attributes
      */
     public function __construct(Expr $value = null, Expr $key = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->key = $key;
         $this->value = $value;
     }

lib/PhpParser/Node/FunctionLike.php

@@ -23,7 +23,7 @@ interface FunctionLike extends Node
     /**
      * Get the declared return type or null
      *
-     * @return null|Identifier|Node\Name|Node\NullableType
+     * @return null|Identifier|Node\Name|Node\NullableType|Node\UnionType
      */
     public function getReturnType();
 

lib/PhpParser/Node/Identifier.php

@@ -25,7 +25,7 @@ class Identifier extends NodeAbstract
      * @param array  $attributes Additional attributes
      */
     public function __construct(string $name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->name = $name;
     }
 

lib/PhpParser/Node/Name.php

@@ -6,9 +6,7 @@ use PhpParser\NodeAbstract;
 
 class Name extends NodeAbstract
 {
-    /**
-     * @var string[] Parts of the name
-     */
+    /** @var string[] Parts of the name */
     public $parts;
 
     private static $specialClassNames = [
@@ -24,7 +22,7 @@ class Name extends NodeAbstract
      * @param array                $attributes Additional attributes
      */
     public function __construct($name, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->parts = self::prepareName($name);
     }
 
@@ -237,7 +235,7 @@ class Name extends NodeAbstract
             'Expected string, array of parts or Name instance'
         );
     }
-    
+
     public function getType() : string {
         return 'Name';
     }

lib/PhpParser/Node/NullableType.php

@@ -16,7 +16,7 @@ class NullableType extends NodeAbstract
      * @param array                  $attributes Additional attributes
      */
     public function __construct($type, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->type = \is_string($type) ? new Identifier($type) : $type;
     }
 

lib/PhpParser/Node/Param.php

@@ -6,7 +6,7 @@ use PhpParser\NodeAbstract;
 
 class Param extends NodeAbstract
 {
-    /** @var null|Identifier|Name|NullableType Typehint */
+    /** @var null|Identifier|Name|NullableType|UnionType Type declaration */
     public $type;
     /** @var bool Whether parameter is passed by reference */
     public $byRef;
@@ -20,18 +20,18 @@ class Param extends NodeAbstract
     /**
      * Constructs a parameter node.
      *
-     * @param Expr\Variable|Expr\Error      $var        Parameter variable
-     * @param null|Expr                     $default    Default value
-     * @param null|string|Name|NullableType $type       Typehint
-     * @param bool                          $byRef      Whether is passed by reference
-     * @param bool                          $variadic   Whether this is a variadic argument
-     * @param array                         $attributes Additional attributes
+     * @param Expr\Variable|Expr\Error                           $var        Parameter variable
+     * @param null|Expr                                          $default    Default value
+     * @param null|string|Identifier|Name|NullableType|UnionType $type       Type declaration
+     * @param bool                                               $byRef      Whether is passed by reference
+     * @param bool                                               $variadic   Whether this is a variadic argument
+     * @param array                                              $attributes Additional attributes
      */
     public function __construct(
         $var, Expr $default = null, $type = null,
         bool $byRef = false, bool $variadic = false, array $attributes = []
     ) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->type = \is_string($type) ? new Identifier($type) : $type;
         $this->byRef = $byRef;
         $this->variadic = $variadic;
@@ -42,7 +42,7 @@ class Param extends NodeAbstract
     public function getSubNodeNames() : array {
         return ['type', 'byRef', 'variadic', 'var', 'default'];
     }
-    
+
     public function getType() : string {
         return 'Param';
     }

lib/PhpParser/Node/Scalar/DNumber.php

@@ -16,7 +16,7 @@ class DNumber extends Scalar
      * @param array $attributes Additional attributes
      */
     public function __construct(float $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
     }
 
@@ -34,6 +34,8 @@ class DNumber extends Scalar
      * @return float The parsed number
      */
     public static function parse(string $str) : float {
+        $str = str_replace('_', '', $str);
+
         // if string contains any of .eE just cast it to float
         if (false !== strpbrk($str, '.eE')) {
             return (float) $str;

lib/PhpParser/Node/Scalar/Encapsed.php

@@ -17,7 +17,7 @@ class Encapsed extends Scalar
      * @param array  $attributes Additional attributes
      */
     public function __construct(array $parts, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->parts = $parts;
     }
 

lib/PhpParser/Node/Scalar/EncapsedStringPart.php

@@ -16,7 +16,7 @@ class EncapsedStringPart extends Scalar
      * @param array  $attributes Additional attributes
      */
     public function __construct(string $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
     }
 

lib/PhpParser/Node/Scalar/LNumber.php

@@ -23,7 +23,7 @@ class LNumber extends Scalar
      * @param array $attributes Additional attributes
      */
     public function __construct(int $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
     }
 
@@ -41,6 +41,8 @@ class LNumber extends Scalar
      * @return LNumber The constructed LNumber, including kind attribute
      */
     public static function fromString(string $str, array $attributes = [], bool $allowInvalidOctal = false) : LNumber {
+        $str = str_replace('_', '', $str);
+
         if ('0' !== $str[0] || '0' === $str) {
             $attributes['kind'] = LNumber::KIND_DEC;
             return new LNumber((int) $str, $attributes);

lib/PhpParser/Node/Scalar/MagicConst.php

@@ -12,7 +12,7 @@ abstract class MagicConst extends Scalar
      * @param array $attributes Additional attributes
      */
     public function __construct(array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
     }
 
     public function getSubNodeNames() : array {

lib/PhpParser/Node/Scalar/String_.php

@@ -34,7 +34,7 @@ class String_ extends Scalar
      * @param array  $attributes Additional attributes
      */
     public function __construct(string $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->value = $value;
     }
 
@@ -100,7 +100,7 @@ class String_ extends Scalar
                 if (isset(self::$replacements[$str])) {
                     return self::$replacements[$str];
                 } elseif ('x' === $str[0] || 'X' === $str[0]) {
-                    return chr(hexdec($str));
+                    return chr(hexdec(substr($str, 1)));
                 } elseif ('u' === $str[0]) {
                     return self::codePointToUtf8(hexdec($matches[2]));
                 } else {
@@ -134,7 +134,7 @@ class String_ extends Scalar
         }
         throw new Error('Invalid UTF-8 codepoint escape sequence: Codepoint too large');
     }
-    
+
     public function getType() : string {
         return 'Scalar_String';
     }

lib/PhpParser/Node/Stmt/Break_.php

@@ -16,7 +16,7 @@ class Break_ extends Node\Stmt
      * @param array          $attributes Additional attributes
      */
     public function __construct(Node\Expr $num = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->num = $num;
     }
 

lib/PhpParser/Node/Stmt/Case_.php

@@ -19,7 +19,7 @@ class Case_ extends Node\Stmt
      * @param array          $attributes Additional attributes
      */
     public function __construct($cond, array $stmts = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->cond = $cond;
         $this->stmts = $stmts;
     }

lib/PhpParser/Node/Stmt/Catch_.php

@@ -9,7 +9,7 @@ class Catch_ extends Node\Stmt
 {
     /** @var Node\Name[] Types of exceptions to catch */
     public $types;
-    /** @var Expr\Variable Variable for exception */
+    /** @var Expr\Variable|null Variable for exception */
     public $var;
     /** @var Node\Stmt[] Statements */
     public $stmts;
@@ -17,15 +17,15 @@ class Catch_ extends Node\Stmt
     /**
      * Constructs a catch node.
      *
-     * @param Node\Name[]   $types      Types of exceptions to catch
-     * @param Expr\Variable $var        Variable for exception
-     * @param Node\Stmt[]   $stmts      Statements
-     * @param array         $attributes Additional attributes
+     * @param Node\Name[]           $types      Types of exceptions to catch
+     * @param Expr\Variable|null    $var        Variable for exception
+     * @param Node\Stmt[]           $stmts      Statements
+     * @param array                 $attributes Additional attributes
      */
     public function __construct(
-        array $types, Expr\Variable $var, array $stmts = [], array $attributes = []
+        array $types, Expr\Variable $var = null, array $stmts = [], array $attributes = []
     ) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->types = $types;
         $this->var = $var;
         $this->stmts = $stmts;
@@ -34,7 +34,7 @@ class Catch_ extends Node\Stmt
     public function getSubNodeNames() : array {
         return ['types', 'var', 'stmts'];
     }
-    
+
     public function getType() : string {
         return 'Stmt_Catch';
     }

lib/PhpParser/Node/Stmt/ClassConst.php

@@ -19,7 +19,7 @@ class ClassConst extends Node\Stmt
      * @param array         $attributes Additional attributes
      */
     public function __construct(array $consts, int $flags = 0, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->flags = $flags;
         $this->consts = $consts;
     }

lib/PhpParser/Node/Stmt/ClassLike.php

@@ -14,6 +14,65 @@ abstract class ClassLike extends Node\Stmt
     /** @var Node\Stmt[] Statements */
     public $stmts;
 
+    /**
+     * @return TraitUse[]
+     */
+    public function getTraitUses() : array {
+        $traitUses = [];
+        foreach ($this->stmts as $stmt) {
+            if ($stmt instanceof TraitUse) {
+                $traitUses[] = $stmt;
+            }
+        }
+        return $traitUses;
+    }
+
+    /**
+     * @return ClassConst[]
+     */
+    public function getConstants() : array {
+        $constants = [];
+        foreach ($this->stmts as $stmt) {
+            if ($stmt instanceof ClassConst) {
+                $constants[] = $stmt;
+            }
+        }
+        return $constants;
+    }
+
+    /**
+     * @return Property[]
+     */
+    public function getProperties() : array {
+        $properties = [];
+        foreach ($this->stmts as $stmt) {
+            if ($stmt instanceof Property) {
+                $properties[] = $stmt;
+            }
+        }
+        return $properties;
+    }
+
+    /**
+     * Gets property with the given name defined directly in this class/interface/trait.
+     *
+     * @param string $name Name of the property
+     *
+     * @return Property|null Property node or null if the property does not exist
+     */
+    public function getProperty(string $name) {
+        foreach ($this->stmts as $stmt) {
+            if ($stmt instanceof Property) {
+                foreach ($stmt->props as $prop) {
+                    if ($prop instanceof PropertyProperty && $name === $prop->name->toString()) {
+                        return $stmt;
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
     /**
      * Gets all methods defined directly in this class/interface/trait
      *

lib/PhpParser/Node/Stmt/ClassMethod.php

@@ -15,7 +15,7 @@ class ClassMethod extends Node\Stmt implements FunctionLike
     public $name;
     /** @var Node\Param[] Parameters */
     public $params;
-    /** @var null|Node\Identifier|Node\Name|Node\NullableType Return type */
+    /** @var null|Node\Identifier|Node\Name|Node\NullableType|Node\UnionType Return type */
     public $returnType;
     /** @var Node\Stmt[]|null Statements */
     public $stmts;
@@ -51,7 +51,7 @@ class ClassMethod extends Node\Stmt implements FunctionLike
      * @param array $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->flags = $subNodes['flags'] ?? $subNodes['type'] ?? 0;
         $this->byRef = $subNodes['byRef'] ?? false;
         $this->name = \is_string($name) ? new Node\Identifier($name) : $name;

lib/PhpParser/Node/Stmt/Class_.php

@@ -35,7 +35,7 @@ class Class_ extends ClassLike
      * @param array       $attributes Additional attributes
      */
     public function __construct($name, array $subNodes = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->flags = $subNodes['flags'] ?? $subNodes['type'] ?? 0;
         $this->name = \is_string($name) ? new Node\Identifier($name) : $name;
         $this->extends = $subNodes['extends'] ?? null;
@@ -98,7 +98,7 @@ class Class_ extends ClassLike
             throw new Error('Cannot use the final modifier on an abstract class member');
         }
     }
-    
+
     public function getType() : string {
         return 'Stmt_Class';
     }

lib/PhpParser/Node/Stmt/Const_.php

@@ -16,7 +16,7 @@ class Const_ extends Node\Stmt
      * @param array         $attributes Additional attributes
      */
     public function __construct(array $consts, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->consts = $consts;
     }
 

lib/PhpParser/Node/Stmt/Continue_.php

@@ -16,7 +16,7 @@ class Continue_ extends Node\Stmt
      * @param array          $attributes Additional attributes
      */
     public function __construct(Node\Expr $num = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->num = $num;
     }
 

lib/PhpParser/Node/Stmt/DeclareDeclare.php

@@ -19,7 +19,7 @@ class DeclareDeclare extends Node\Stmt
      * @param array                  $attributes Additional attributes
      */
     public function __construct($key, Node\Expr $value, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->key = \is_string($key) ? new Node\Identifier($key) : $key;
         $this->value = $value;
     }

lib/PhpParser/Node/Stmt/Declare_.php

@@ -19,7 +19,7 @@ class Declare_ extends Node\Stmt
      * @param array            $attributes Additional attributes
      */
     public function __construct(array $declares, array $stmts = null, array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->declares = $declares;
         $this->stmts = $stmts;
     }

lib/PhpParser/Node/Stmt/Do_.php

@@ -19,7 +19,7 @@ class Do_ extends Node\Stmt
      * @param array       $attributes Additional attributes
      */
     public function __construct(Node\Expr $cond, array $stmts = [], array $attributes = []) {
-        parent::__construct($attributes);
+        $this->attributes = $attributes;
         $this->cond = $cond;
         $this->stmts = $stmts;
     }