Prepare PHP-Parser 1.4.0 release

This reverts commit 91295a0790.

This should no longer be necessary.

Conflicts:
	test/PhpParser/ParserTest.php

CHANGELOG.md

@@ -1,8 +1,48 @@
-Version 1.2.3-dev
+Version 1.4.1-dev
 -----------------
 
 Nothing yet.
 
+Version 1.4.0 (2015-07-14)
+--------------------------
+
+### Added
+
+* Added interface `PhpParser\Node\FunctionLike`, which is implemented by `Stmt\ClassMethod`,
+  `Stmt\Function_` and `Expr\Closure` nodes. This interface provides getters for their common
+  subnodes.
+* Added `Node\Stmt\ClassLike::getMethod()` to look up a specific method on a class/interface/trait.
+
+### Fixed
+
+* Fixed `isPublic()` return value for implicitly public properties and methods that define and
+  additional modifier like `static` or `abstract`.
+* Properties are now accepted by the trait builder.
+* Fixed `__HALT_COMPILER_OFFSET__` support on HHVM.
+
+Version 1.3.0 (2015-05-02)
+--------------------------
+
+### Added
+
+* Errors can now store the attributes of the node/token where the error occurred. Previously only the start line was
+  stored.
+* If file positions are enabled in the lexer, errors can now provide column information if it is available. See
+  [documentation](https://github.com/nikic/PHP-Parser/blob/master/doc/component/Error.markdown#column-information).
+* The parser now provides an experimental error recovery mode, which can be enabled by disabling the `throwOnError`
+  parser option. In this mode the parser will try to construct a partial AST even if the code is not valid PHP. See
+  [documentation](https://github.com/nikic/PHP-Parser/blob/master/doc/component/Error.markdown#error-recovery).
+* Added support for PHP 7 `yield from` expression. It is represented by `Expr\YieldFrom`.
+* Added support for PHP 7 anonymous classes. These are represented by ordinary `Stmt\Class_` nodes with the name set to
+  `null`. Furthermore this implies that `Expr\New_` can now contain a `Stmt\Class_` in its `class` subnode.
+
+### Fixed
+
+* Fixed registration of PHP 7 aliases, for the case where the old name was used before the new name.
+* Fixed handling of precedence when pretty-printing `print` expressions.
+* Floating point numbers are now pretty-printed with a higher precision.
+* Checks for special class names like `self` are now case-insensitive.
+
 Version 1.2.2 (2015-04-03)
 --------------------------
 

README.md

@@ -4,7 +4,7 @@ PHP Parser
 This is a PHP 5.2 to PHP 5.6 parser written in PHP. Its purpose is to simplify static code analysis and
 manipulation.
 
-[**Documentation for version 1.x**][doc_master] (stable; for running on PHP >= 5.3).
+[**Documentation for version 1.x**][doc_1_x] (stable; for running on PHP >= 5.3).
 
 [Documentation for version 0.9.x][doc_0_9] (unsupported; for running on PHP 5.2).
 
@@ -22,7 +22,7 @@ hello\world('foo', 'bar' . 'baz');
 
 You'll get a syntax tree looking roughly like this:
 
-```
+```php
 array(
     0: Stmt_Echo(
         exprs: array(
@@ -84,4 +84,4 @@ Component documentation:
  1. [Lexer](doc/component/Lexer.markdown)
 
  [doc_0_9]: https://github.com/nikic/PHP-Parser/tree/0.9/doc
- [doc_master]: https://github.com/nikic/PHP-Parser/tree/master/doc
+ [doc_1_x]: https://github.com/nikic/PHP-Parser/tree/1.x/doc

bin/php-parse.php

@@ -10,7 +10,7 @@ ini_set('xdebug.var_display_max_children', -1);
 ini_set('xdebug.var_display_max_data', -1);
 ini_set('xdebug.var_display_max_depth', -1);
 
-list($operations, $files) = parseArgs($argv);
+list($operations, $files, $attributes) = parseArgs($argv);
 
 /* Dump nodes by default */
 if (empty($operations)) {
@@ -21,7 +21,10 @@ if (empty($files)) {
     showHelp("Must specify at least one file.");
 }
 
-$parser = new PhpParser\Parser(new PhpParser\Lexer\Emulative);
+$lexer = new PhpParser\Lexer\Emulative(array('usedAttributes' => array(
+    'startLine', 'endLine', 'startFilePos', 'endFilePos'
+)));
+$parser = new PhpParser\Parser($lexer);
 $dumper = new PhpParser\NodeDumper;
 $prettyPrinter = new PhpParser\PrettyPrinter\Standard;
 $serializer = new PhpParser\Serializer\XML;
@@ -45,7 +48,17 @@ foreach ($files as $file) {
     try {
         $stmts = $parser->parse($code);
     } catch (PhpParser\Error $e) {
-        die("==> Parse Error: {$e->getMessage()}\n");
+        if ($attributes['with-column-info'] && $e->hasColumnInfo()) {
+            $startLine = $e->getStartLine();
+            $endLine = $e->getEndLine();
+            $startColumn = $e->getStartColumn($code);
+            $endColumn   = $e->getEndColumn($code);
+            $message .= $e->getRawMessage() . " from $startLine:$startColumn to $endLine:$endColumn";
+        } else {
+            $message = $e->getMessage();
+        }
+
+        die($message . "\n");
     }
 
     foreach ($operations as $operation) {
@@ -71,24 +84,20 @@ foreach ($files as $file) {
 function showHelp($error) {
     die($error . "\n\n" .
         <<<OUTPUT
-Usage:
-
-    php php-parse.php [operations] file1.php [file2.php ...]
-
-The file arguments can also be replaced with a code string:
-
-    php php-parse.php [operations] "<?php code"
+Usage: php php-parse.php [operations] file1.php [file2.php ...]
+   or: php php-parse.php [operations] "<?php code"
+Turn PHP source code into an abstract syntax tree.
 
 Operations is a list of the following options (--dump by default):
 
-    --dump           -d  Dump nodes using NodeDumper
-    --pretty-print   -p  Pretty print file using PrettyPrinter\Standard
+    -d, --dump              Dump nodes using NodeDumper
+    -p, --pretty-print      Pretty print file using PrettyPrinter\Standard
         --serialize-xml     Serialize nodes using Serializer\XML
         --var-dump          var_dump() nodes (for exact structure)
-    --resolve-names  -N  Resolve names using NodeVisitor\NameResolver
+    -N, --resolve-names     Resolve names using NodeVisitor\NameResolver
+    -c, --with-column-info  Show column-numbers for errors (if available)
 
 Example:
-
     php php-parse.php -d -p -N -d file.php
 
     Dumps nodes, pretty prints them, then resolves names and dumps them again.
@@ -101,6 +110,9 @@ OUTPUT
 function parseArgs($args) {
     $operations = array();
     $files = array();
+    $attributes = array(
+        'with-column-info' => false,
+    );
 
     array_shift($args);
     $parseOptions = true;
@@ -129,6 +141,10 @@ function parseArgs($args) {
             case '-N';
                 $operations[] = 'resolve-names';
                 break;
+            case '--with-column-info':
+            case '-c';
+                $attributes['with-column-info'] = true;
+                break;
             case '--':
                 $parseOptions = false;
                 break;
@@ -141,5 +157,5 @@ function parseArgs($args) {
         }
     }
 
-    return array($operations, $files);
+    return array($operations, $files, $attributes);
 }

composer.json

@@ -18,7 +18,7 @@
     },
     "extra": {
         "branch-alias": {
-            "dev-master": "1.2-dev"
+            "dev-master": "1.4-dev"
         }
     }
 }

doc/component/Error.markdown

@@ -0,0 +1,77 @@
+Error handling
+==============
+
+Errors during parsing or analysis are represented using the `PhpParser\Error` exception class. In addition to an error
+message, an error can also store additional information about the location the error occurred at.
+
+How much location information is available depends on the origin of the error and how many lexer attributes have been
+enabled. At a minimum the start line of the error is usually available.
+
+Column information
+------------------
+
+In order to receive information about not only the line, but also the column span an error occurred at, the file
+position attributes in the lexer need to be enabled:
+
+```php
+$lexer = new PhpParser\Lexer(array(
+    'usedAttributes' => array('comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'),
+));
+$parser = new PhpParser\Parser($lexer);
+
+try {
+    $stmts = $parser->parse($code);
+    // ...
+} catch (PhpParser\Error $e) {
+    // ...
+}
+```
+
+Before using column information its availability needs to be checked with `$e->hasColumnInfo()`, as the precise
+location of an error cannot always be determined. The methods for retrieving column information also have to be passed
+the source code of the parsed file. An example for printing an error:
+
+```php
+if ($e->hasColumnInfo()) {
+    echo $e->getRawMessage() . ' from ' . $e->getStartLine() . ':' . $e->getStartColumn($code)
+        . ' to ' . $e->getEndLine() . ':' . $e->getEndColumn($code);
+} else {
+    echo $e->getMessage();
+}
+```
+
+Both line numbers and column numbers are 1-based. EOF errors will be located at the position one past the end of the
+file.
+
+Error recovery
+--------------
+
+> **EXPERIMENTAL**
+
+By default the parser will throw an exception upon encountering the first error during parsing. An alternative mode is
+also supported, in which the parser will remember the error, but try to continue parsing the rest of the source code.
+
+To enable this mode the `throwOnError` parser option needs to be disabled. Any errors that occurred during parsing can
+then be retrieved using `$parser->getErrors()`. The `$parser->parse()` method will either return a partial syntax tree
+or `null` if recovery fails.
+
+A usage example:
+
+```php
+$parser = new PhpParser\Parser(new PhpParser\Lexer, array(
+    'throwOnError' => false,
+));
+
+$stmts = $parser->parse($code);
+$errors = $parser->getErrors();
+
+foreach ($errors as $error) {
+    // $error is an ordinary PhpParser\Error
+}
+
+if (null !== $stmts) {
+    // $stmts is a best-effort partial AST
+}
+```
+
+The error recovery implementation is experimental -- it currently won't be able to recover from many types of errors.

grammar/README.md

@@ -13,8 +13,7 @@ The `.phpy` file is a normal grammer in `kmyacc` (`yacc`) style, with some trans
 applied to it:
 
  * Nodes are created using the syntax `Name[..., ...]`. This is transformed into
-   `new Node\Name(..., ..., $attributes)`
- * `Name::abc` is transformed to `Node\Name::abc`
+   `new Name(..., ..., attributes())`
  * Some function-like constructs are resolved (see `rebuildParser.php` for a list)
  * Associative arrays are written as `[key: value, ...]`, which is transformed to
    `array('key' => value, ...)`
@@ -25,5 +24,5 @@ Building the parser
 In order to rebuild the parser, you need [moriyoshi's fork of kmyacc](https://github.com/moriyoshi/kmyacc-forked).
 After you compiled/installed it, run the `rebuildParser.php` script.
 
-By default only the `Parser.php` is built. If you want to additionally build `Parser/Debug.php` and `y.output` run the
+By default only the `Parser.php` is built. If you want to additionally emit debug symbols and create `y.output`, run the
 script with `--debug`. If you want to retain the preprocessed grammar pass `--keep-tmp-grammar`.

grammar/kmyacc.php.parser

@@ -2,11 +2,11 @@
 $meta #
 #semval($) $this->semValue
 #semval($,%t) $this->semValue
-#semval(%n) $this->semStack[$this->stackPos-(%l-%n)]
-#semval(%n,%t) $this->semStack[$this->stackPos-(%l-%n)]
-#include;
+#semval(%n) $this->stackPos-(%l-%n)
+#semval(%n,%t) $this->stackPos-(%l-%n)
 
 namespace PhpParser;
+#include;
 
 /* This is an automatically GENERATED file, which should not be manually edited.
  * Instead edit one of the following:
@@ -20,7 +20,8 @@ class Parser extends ParserAbstract
     protected $actionTableSize = #(YYLAST);
     protected $gotoTableSize = #(YYGLAST);
 
-    protected $invalidToken = #(YYBADCH);
+    protected $invalidSymbol = #(YYBADCH);
+    protected $errorSymbol = #(YYINTERRTOK);
     protected $defaultAction = #(YYDEFAULT);
     protected $unexpectedTokenRule = #(YYUNEXPECTED);
 
@@ -86,7 +87,7 @@ class Parser extends ParserAbstract
 #endif
 #reduce
 
-    protected function reduceRule%n($attributes) {
+    protected function reduceRule%n() {
         %b
     }
 #noact

grammar/rebuildParser.php

@@ -39,10 +39,10 @@ echo 'Building temporary preproprocessed grammar file.', "\n";
 
 $grammarCode = file_get_contents($grammarFile);
 
-$grammarCode = resolveConstants($grammarCode);
 $grammarCode = resolveNodes($grammarCode);
 $grammarCode = resolveMacros($grammarCode);
 $grammarCode = resolveArrays($grammarCode);
+$grammarCode = resolveStackAccess($grammarCode);
 
 file_put_contents($tmpGrammarFile, $grammarCode);
 
@@ -52,7 +52,12 @@ echo "Building parser.\n";
 $output = trim(shell_exec("$kmyacc $additionalArgs -l -m $skeletonFile $tmpGrammarFile 2>&1"));
 echo "Output: \"$output\"\n";
 
-moveFileWithDirCheck($tmpResultFile, $parserResultFile);
+$resultCode = file_get_contents($tmpResultFile);
+$resultCode = removeTrailingWhitespace($resultCode);
+
+ensureDirExists(dirname($parserResultFile));
+file_put_contents($parserResultFile, $resultCode);
+unlink($tmpResultFile);
 
 if (!$optionKeepTmpGrammar) {
     unlink($tmpGrammarFile);
@@ -62,10 +67,6 @@ if (!$optionKeepTmpGrammar) {
 /// Preprocessing functions ///
 ///////////////////////////////
 
-function resolveConstants($code) {
-    return preg_replace('~[A-Z][a-zA-Z_\\\\]++::~', 'Node\\\\$0', $code);
-}
-
 function resolveNodes($code) {
     return preg_replace_callback(
         '~(?<name>[A-Z][a-zA-Z_\\\\]++)\s*' . PARAMS . '~',
@@ -83,7 +84,7 @@ function resolveNodes($code) {
                 $paramCode .= $param . ', ';
             }
 
-            return 'new Node\\' . $matches['name'] . '(' . $paramCode . '$attributes)';
+            return 'new ' . $matches['name'] . '(' . $paramCode . 'attributes())';
         },
         $code
     );
@@ -102,10 +103,9 @@ function resolveMacros($code) {
                 $matches['args']
             );
 
-            if ('error' == $name) {
-                assertArgs(1, $args, $name);
-
-                return 'throw new Error(' . $args[0] . ')';
+            if ('attributes' == $name) {
+                assertArgs(0, $args, $name);
+                return '$this->startAttributeStack[#1] + $this->endAttributes';
             }
 
             if ('init' == $name) {
@@ -148,7 +148,7 @@ function resolveMacros($code) {
                 return 'foreach (' . $args[0] . ' as &$s) { if (is_string($s)) { $s = Node\Scalar\String_::parseEscapeSequences($s, null); } } $s = preg_replace(\'~(\r\n|\n|\r)$~\', \'\', $s); if (\'\' === $s) array_pop(' . $args[0] . ');';
             }
 
-            throw new Exception(sprintf('Unknown macro "%s"', $name));
+            return $matches[0];
         },
         $code
     );
@@ -191,12 +191,22 @@ function resolveArrays($code) {
     );
 }
 
-function moveFileWithDirCheck($fromPath, $toPath) {
-    $dir = dirname($toPath);
+function resolveStackAccess($code) {
+    $code = preg_replace('/\$\d+/', '$this->semStack[$0]', $code);
+    $code = preg_replace('/#(\d+)/', '$$1', $code);
+    return $code;
+}
+
+function removeTrailingWhitespace($code) {
+    $lines = explode("\n", $code);
+    $lines = array_map('rtrim', $lines);
+    return implode("\n", $lines);
+}
+
+function ensureDirExists($dir) {
     if (!is_dir($dir)) {
         mkdir($dir, 0777, true);
     }
-    rename($fromPath, $toPath);
 }
 
 //////////////////////////////

grammar/zend_language_parser.phpy

@@ -8,6 +8,7 @@
 %left T_LOGICAL_AND
 %right T_PRINT
 %right T_YIELD
+%right T_YIELD_FROM
 %left '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL
 %left '?' ':'
 %right T_COALESCE
@@ -110,6 +111,13 @@
 %token T_NS_SEPARATOR
 %token T_ELLIPSIS
 
+%{
+use PhpParser\Node\Expr;
+use PhpParser\Node\Name;
+use PhpParser\Node\Scalar;
+use PhpParser\Node\Stmt;
+%}
+
 %%
 
 start:
@@ -163,7 +171,7 @@ constant_declaration_list:
 ;
 
 constant_declaration:
-    T_STRING '=' static_scalar                              { $$ = Const_[$1, $3]; }
+    T_STRING '=' static_scalar                              { $$ = Node\Const_[$1, $3]; }
 ;
 
 inner_statement_list:
@@ -175,7 +183,8 @@ inner_statement:
       statement                                             { $$ = $1; }
     | function_declaration_statement                        { $$ = $1; }
     | class_declaration_statement                           { $$ = $1; }
-    | T_HALT_COMPILER                                       { error('__HALT_COMPILER() can only be used from the outermost scope'); }
+    | T_HALT_COMPILER
+          { throw new Error('__HALT_COMPILER() can only be used from the outermost scope', attributes()); }
 ;
 
 statement:
@@ -213,6 +222,7 @@ statement:
     | T_THROW expr ';'                                      { $$ = Stmt\Throw_[$2]; }
     | T_GOTO T_STRING ';'                                   { $$ = Stmt\Goto_[$2]; }
     | T_STRING ':'                                          { $$ = Stmt\Label[$1]; }
+    | error                                                 { $$ = array(); /* means: no statement */ }
 ;
 
 catches:
@@ -382,9 +392,9 @@ non_empty_parameter_list:
 
 parameter:
       optional_param_type optional_ref optional_ellipsis T_VARIABLE
-          { $$ = Param[parseVar($4), null, $1, $2, $3]; }
+          { $$ = Node\Param[parseVar($4), null, $1, $2, $3]; }
     | optional_param_type optional_ref optional_ellipsis T_VARIABLE '=' static_scalar
-          { $$ = Param[parseVar($4), $6, $1, $2, $3]; }
+          { $$ = Node\Param[parseVar($4), $6, $1, $2, $3]; }
 ;
 
 type:
@@ -406,7 +416,7 @@ optional_return_type:
 argument_list:
       '(' ')'                                               { $$ = array(); }
     | '(' non_empty_argument_list ')'                       { $$ = $2; }
-    | '(' yield_expr ')'                                    { $$ = array(Arg[$2, false, false]); }
+    | '(' yield_expr ')'                                    { $$ = array(Node\Arg[$2, false, false]); }
 ;
 
 non_empty_argument_list:
@@ -415,9 +425,9 @@ non_empty_argument_list:
 ;
 
 argument:
-      expr                                                  { $$ = Arg[$1, false, false]; }
-    | '&' variable                                          { $$ = Arg[$2, true, false]; }
-    | T_ELLIPSIS expr                                       { $$ = Arg[$2, false, true]; }
+      expr                                                  { $$ = Node\Arg[$1, false, false]; }
+    | '&' variable                                          { $$ = Node\Arg[$2, true, false]; }
+    | T_ELLIPSIS expr                                       { $$ = Node\Arg[$2, false, true]; }
 ;
 
 global_var_list:
@@ -615,6 +625,7 @@ expr:
     | '`' backticks_expr '`'                                { $$ = Expr\ShellExec[$2]; }
     | T_PRINT expr                                          { $$ = Expr\Print_[$2]; }
     | T_YIELD                                               { $$ = Expr\Yield_[null, null]; }
+    | T_YIELD_FROM expr                                     { $$ = Expr\YieldFrom[$2]; }
     | T_FUNCTION optional_ref '(' parameter_list ')' lexical_vars optional_return_type
       '{' inner_statement_list '}'
           { $$ = Expr\Closure[[static: false, byRef: $2, params: $4, uses: $6, returnType: $7, stmts: $9]]; }
@@ -647,8 +658,14 @@ scalar_dereference:
     /* alternative array syntax missing intentionally */
 ;
 
+anonymous_class:
+      T_CLASS ctor_arguments extends_from implements_list '{' class_statement_list '}'
+          { $$ = array(Stmt\Class_[null, [type: 0, extends: $3, implements: $4, stmts: $6]], $2); }
+
 new_expr:
       T_NEW class_name_reference ctor_arguments             { $$ = Expr\New_[$2, $3]; }
+    | T_NEW anonymous_class
+          { list($class, $ctorArgs) = $2; $$ = Expr\New_[$class, $ctorArgs]; }
 ;
 
 lexical_vars:

lib/PhpParser/Autoloader.php

@@ -36,12 +36,14 @@ class Autoloader
     static public function autoload($class) {
         if (0 === strpos($class, 'PhpParser\\')) {
             if (isset(self::$php7AliasesOldToNew[$class])) {
-                // Old class name was used, register alias to new one (which will
-                // be autoloaded, if it wasn't yet).
-                self::registerPhp7Alias(self::$php7AliasesOldToNew[$class], $class);
+                if (self::$runningOnPhp7) {
                     return;
                 }
 
+                // Load the new class, alias will be registered afterwards
+                $class = self::$php7AliasesOldToNew[$class];
+            }
+
             $fileName = dirname(__DIR__) . '/' . strtr($class, '\\', '/') . '.php';
             if (file_exists($fileName)) {
                 require $fileName;
@@ -50,7 +52,9 @@ class Autoloader
             if (isset(self::$php7AliasesNewToOld[$class])) {
                 // New class name was used, register alias for old one, otherwise
                 // it won't be usable in "instanceof" and other non-autoloading places.
-                self::registerPhp7Alias($class, self::$php7AliasesNewToOld[$class]);
+                if (!self::$runningOnPhp7) {
+                    class_alias($class, self::$php7AliasesNewToOld[$class]);
+                }
             }
         } else if (0 === strpos($class, 'PHPParser_')) {
             if (isset(self::$nonNamespacedAliases[$class])) {
@@ -60,14 +64,6 @@ class Autoloader
         }
     }
 
-    private static function registerPhp7Alias($old, $new) {
-        // Registering these aliases would throw a fatal error on PHP 7,
-        // we want to avoid that.
-        if (!self::$runningOnPhp7) {
-            class_alias($old, $new);
-        }
-    }
-
     private static function registerNonNamespacedAliases() {
         foreach (self::$nonNamespacedAliases as $old => $new) {
             class_alias($new, $old);

lib/PhpParser/Builder/Trait_.php

@@ -9,6 +9,7 @@ use PhpParser\Node\Stmt;
 class Trait_ extends Declaration
 {
     protected $name;
+    protected $properties = array();
     protected $methods = array();
 
     /**
@@ -29,12 +30,15 @@ class Trait_ extends Declaration
      */
     public function addStmt($stmt) {
         $stmt = $this->normalizeNode($stmt);
-        if (!$stmt instanceof Stmt\ClassMethod) {
+
+        if ($stmt instanceof Stmt\Property) {
+            $this->properties[] = $stmt;
+        } else if ($stmt instanceof Stmt\ClassMethod) {
+            $this->methods[] = $stmt;
+        } else {
             throw new \LogicException(sprintf('Unexpected node of type "%s"', $stmt->getType()));
         }
 
-        $this->methods[] = $stmt;
-
         return $this;
     }
 
@@ -44,6 +48,8 @@ class Trait_ extends Declaration
      * @return Stmt\Trait_ The built interface node
      */
     public function getNode() {
-        return new Stmt\Trait_($this->name, $this->methods, $this->attributes);
+        return new Stmt\Trait_(
+            $this->name, array_merge($this->properties, $this->methods), $this->attributes
+        );
     }
 }

lib/PhpParser/Builder/Use_.php

@@ -36,8 +36,12 @@ class Use_ extends BuilderAbstract {
         $this->alias = $alias;
         return $this;
     }
-    public function __call($method, $args) {
-        return call_user_func_array(array($this, $method . '_'), $args);
+    public function __call($name, $args) {
+        if (method_exists($this, $name . '_')) {
+            return call_user_func_array(array($this, $name . '_'), $args);
+        }
+
+        throw new \LogicException(sprintf('Method "%s" does not exist', $name));
     }
 
     /**

lib/PhpParser/Error.php

@@ -5,17 +5,22 @@ namespace PhpParser;
 class Error extends \RuntimeException
 {
     protected $rawMessage;
-    protected $rawLine;
+    protected $attributes;
 
     /**
      * Creates an Exception signifying a parse error.
      *
      * @param string    $message    Error message
-     * @param int    $line    Error line in PHP file
+     * @param array|int $attributes Attributes of node/token where error occurred
+     *                              (or start line of error -- deprecated)
      */
-    public function __construct($message, $line = -1) {
+    public function __construct($message, $attributes = array()) {
         $this->rawMessage = (string) $message;
-        $this->rawLine    = (int) $line;
+        if (is_array($attributes)) {
+            $this->attributes = $attributes;
+        } else {
+            $this->attributes = array('startLine' => $attributes);
+        }
         $this->updateMessage();
     }
 
@@ -28,6 +33,34 @@ class Error extends \RuntimeException
         return $this->rawMessage;
     }
 
+    /**
+     * Gets the line the error starts in.
+     *
+     * @return int Error start line
+     */
+    public function getStartLine() {
+        return isset($this->attributes['startLine']) ? $this->attributes['startLine'] : -1;
+    }
+
+    /**
+     * Gets the line the error ends in.
+     *
+     * @return int Error end line
+     */
+    public function getEndLine() {
+        return isset($this->attributes['endLine']) ? $this->attributes['endLine'] : -1;
+    }
+
+
+    /**
+     * Gets the attributes of the node/token the error occurred at.
+     *
+     * @return array
+     */
+    public function getAttributes() {
+        return $this->attributes;
+    }
+
     /**
      * Sets the line of the PHP file the error occurred in.
      *
@@ -39,22 +72,65 @@ class Error extends \RuntimeException
     }
 
     /**
-     * Gets the error line in the PHP file.
+     * Sets the line the error starts in.
      *
-     * @return int Error line in the PHP file
+     * @param int $line Error start line
      */
-    public function getRawLine() {
-        return $this->rawLine;
+    public function setStartLine($line) {
+        $this->attributes['startLine'] = (int) $line;
+        $this->updateMessage();
     }
 
     /**
-     * Sets the line of the PHP file the error occurred in.
+     * Returns whether the error has start and end column information.
      *
-     * @param int $line Error line in the PHP file
+     * For column information enable the startFilePos and endFilePos in the lexer options.
+     *
+     * @return bool
      */
-    public function setRawLine($line) {
-        $this->rawLine = (int) $line;
-        $this->updateMessage();
+    public function hasColumnInfo() {
+        return isset($this->attributes['startFilePos']) && isset($this->attributes['endFilePos']);
+    }
+
+    /**
+     * Gets the start column (1-based) into the line where the error started.
+     *
+     * @param string $code Source code of the file
+     * @return int
+     */
+    public function getStartColumn($code) {
+        if (!$this->hasColumnInfo()) {
+            throw new \RuntimeException('Error does not have column information');
+        }
+
+        return $this->toColumn($code, $this->attributes['startFilePos']);
+    }
+
+    /**
+     * Gets the end column (1-based) into the line where the error ended.
+     *
+     * @param string $code Source code of the file
+     * @return int
+     */
+    public function getEndColumn($code) {
+        if (!$this->hasColumnInfo()) {
+            throw new \RuntimeException('Error does not have column information');
+        }
+
+        return $this->toColumn($code, $this->attributes['endFilePos']);
+    }
+
+    private function toColumn($code, $pos) {
+        if ($pos > strlen($code)) {
+            throw new \RuntimeException('Invalid position information');
+        }
+
+        $lineStartPos = strrpos($code, "\n", $pos - strlen($code));
+        if (false === $lineStartPos) {
+            $lineStartPos = -1;
+        }
+
+        return $pos - $lineStartPos;
     }
 
     /**
@@ -63,10 +139,20 @@ class Error extends \RuntimeException
     protected function updateMessage() {
         $this->message = $this->rawMessage;
 
-        if (-1 === $this->rawLine) {
+        if (-1 === $this->getStartLine()) {
             $this->message .= ' on unknown line';
         } else {
-            $this->message .= ' on line ' . $this->rawLine;
+            $this->message .= ' on line ' . $this->getStartLine();
         }
     }
+
+    /** @deprecated Use getStartLine() instead */
+    public function getRawLine() {
+        return $this->getStartLine();
+    }
+
+    /** @deprecated Use setStartLine() instead */
+    public function setRawLine($line) {
+        $this->setStartLine($line);
+    }
 }

lib/PhpParser/Lexer.php

@@ -123,8 +123,13 @@ class Lexer
         $startAttributes = array();
         $endAttributes   = array();
 
-        while (isset($this->tokens[++$this->pos])) {
+        while (1) {
+            if (isset($this->tokens[++$this->pos])) {
                 $token = $this->tokens[$this->pos];
+            } else {
+                // EOF token with ID 0
+                $token = "\0";
+            }
 
             if (isset($this->usedAttributes['startTokenPos'])) {
                 $startAttributes['startTokenPos'] = $this->pos;
@@ -192,10 +197,7 @@ class Lexer
             }
         }
 
-        $startAttributes['startLine'] = $this->line;
-
-        // 0 is the EOF token
-        return 0;
+        throw new \RuntimeException('Reached end of lexer loop');
     }
 
     /**
@@ -218,23 +220,13 @@ class Lexer
      * @return string Remaining text
      */
     public function handleHaltCompiler() {
-        // get the length of the text before the T_HALT_COMPILER token
-        $textBefore = '';
-        for ($i = 0; $i <= $this->pos; ++$i) {
-            if (is_string($this->tokens[$i])) {
-                $textBefore .= $this->tokens[$i];
-            } else {
-                $textBefore .= $this->tokens[$i][1];
-            }
-        }
-
         // text after T_HALT_COMPILER, still including ();
-        $textAfter = substr($this->code, strlen($textBefore));
+        $textAfter = substr($this->code, $this->filePos);
 
         // ensure that it is followed by ();
         // this simplifies the situation, by not allowing any comments
         // in between of the tokens.
-        if (!preg_match('~\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
+        if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
             throw new Error('__HALT_COMPILER must be followed by "();"');
         }
 
@@ -284,6 +276,10 @@ class Lexer
         if (defined('T_ONUMBER')) {
             $tokenMap[T_ONUMBER] = Parser::T_DNUMBER;
         }
+        // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
+        if (defined('T_COMPILER_HALT_OFFSET')) {
+            $tokenMap[T_COMPILER_HALT_OFFSET] = Parser::T_STRING;
+        }
 
         return $tokenMap;
     }

lib/PhpParser/Lexer/Emulative.php

@@ -17,6 +17,7 @@ class Emulative extends \PhpParser\Lexer
     const T_POW_EQUAL  = 1003;
     const T_COALESCE   = 1004;
     const T_SPACESHIP  = 1005;
+    const T_YIELD_FROM = 1006;
 
     const PHP_7_0 = '7.0.0dev';
     const PHP_5_6 = '5.6.0rc1';
@@ -53,6 +54,7 @@ class Emulative extends \PhpParser\Lexer
         }
         $this->tokenMap[self::T_COALESCE] = Parser::T_COALESCE;
         $this->tokenMap[self::T_SPACESHIP] = Parser::T_SPACESHIP;
+        $this->tokenMap[self::T_YIELD_FROM] = Parser::T_YIELD_FROM;
 
         if (version_compare(PHP_VERSION, self::PHP_5_6, '>=')) {
             return;
@@ -91,6 +93,10 @@ class Emulative extends \PhpParser\Lexer
 
         $code = str_replace('??', '~__EMU__COALESCE__~', $code);
         $code = str_replace('<=>', '~__EMU__SPACESHIP__~', $code);
+        $code = preg_replace_callback('(yield[ \n\r\t]+from)', function($matches) {
+            // Encoding $0 in order to preserve exact whitespace
+            return '~__EMU__YIELDFROM__' . bin2hex($matches[0]) . '__~';
+        }, $code);
 
         if (version_compare(PHP_VERSION, self::PHP_5_6, '>=')) {
             return $code;
@@ -127,8 +133,9 @@ class Emulative extends \PhpParser\Lexer
                 if ('BINARY' === $matches[1]) {
                     // the binary number can either be an integer or a double, so return a LNUMBER
                     // or DNUMBER respectively
+                    $isInt = is_int(bindec($matches[2]));
                     $replace = array(
-                        array(is_int(bindec($matches[2])) ? T_LNUMBER : T_DNUMBER, $matches[2], $this->tokens[$i + 1][2])
+                        array($isInt ? T_LNUMBER : T_DNUMBER, $matches[2], $this->tokens[$i + 1][2])
                     );
                 } else if ('ELLIPSIS' === $matches[1]) {
                     $replace = array(
@@ -150,9 +157,13 @@ class Emulative extends \PhpParser\Lexer
                     $replace = array(
                         array(self::T_SPACESHIP, '<=>', $this->tokens[$i + 1][2]),
                     );
+                } else if ('YIELDFROM' === $matches[1]) {
+                    $content = $this->hex2bin($matches[2]);
+                    $replace = array(
+                        array(self::T_YIELD_FROM, $content, $this->tokens[$i + 1][2] - substr_count($content, "\n"))
+                    );
                 } else {
-                    // just ignore all other __EMU__ sequences
-                    continue;
+                    throw new \RuntimeException('Invalid __EMU__ sequence');
                 }
 
                 array_splice($this->tokens, $i, 3, $replace);
@@ -188,11 +199,18 @@ class Emulative extends \PhpParser\Lexer
             return '??';
         } else if ('SPACESHIP' === $matches[1]) {
             return '<=>';
+        } else if ('YIELDFROM' === $matches[1]) {
+            return $this->hex2bin($matches[2]);
         } else {
             return $matches[0];
         }
     }
 
+    private function hex2bin($str) {
+        // TODO Drop when removing support for PHP 5.3
+        return pack('H*', $str);
+    }
+
     public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) {
         $token = parent::getNextToken($value, $startAttributes, $endAttributes);
 
@@ -203,11 +221,9 @@ class Emulative extends \PhpParser\Lexer
             if (isset($this->newKeywords[strtolower($value)])) {
                 return $this->newKeywords[strtolower($value)];
             }
-        // keep track of whether we currently are in an object access (after ->)
-        } elseif (Parser::T_OBJECT_OPERATOR === $token) {
-            $this->inObjectAccess = true;
         } else {
-            $this->inObjectAccess = false;
+            // keep track of whether we currently are in an object access (after ->)
+            $this->inObjectAccess = Parser::T_OBJECT_OPERATOR === $token;
         }
 
         return $token;

lib/PhpParser/Node/Expr/Closure.php

@@ -4,8 +4,9 @@ namespace PhpParser\Node\Expr;
 
 use PhpParser\Node;
 use PhpParser\Node\Expr;
+use PhpParser\Node\FunctionLike;
 
-class Closure extends Expr
+class Closure extends Expr implements FunctionLike
 {
     /** @var bool Whether the closure is static */
     public $static;
@@ -15,7 +16,7 @@ class Closure extends Expr
     public $params;
     /** @var ClosureUse[] use()s */
     public $uses;
-    /** @var null|string|Node\Name[] Return type */
+    /** @var null|string|Node\Name Return type */
     public $returnType;
     /** @var Node[] Statements */
     public $stmts;
@@ -45,4 +46,20 @@ class Closure extends Expr
     public function getSubNodeNames() {
         return array('static', 'byRef', 'params', 'uses', 'returnType', 'stmts');
     }
+
+    public function returnsByRef() {
+        return $this->byRef;
+    }
+
+    public function getParams() {
+        return $this->params;
+    }
+
+    public function getReturnType() {
+        return $this->returnType;
+    }
+
+    public function getStmts() {
+        return $this->stmts;
+    }
 }

lib/PhpParser/Node/Expr/New_.php

@@ -7,7 +7,7 @@ use PhpParser\Node\Expr;
 
 class New_ extends Expr
 {
-    /** @var Node\Name|Expr Class name */
+    /** @var Node\Name|Expr|Node\Stmt\Class_ Class name */
     public $class;
     /** @var Node\Arg[] Arguments */
     public $args;
@@ -15,7 +15,7 @@ class New_ extends Expr
     /**
      * Constructs a function call node.
      *
-     * @param Node\Name|Expr $class      Class name
+     * @param Node\Name|Expr|Node\Stmt\Class_ $class      Class name (or class node for anonymous classes)
      * @param Node\Arg[]                      $args       Arguments
      * @param array                           $attributes Additional attributes
      */

lib/PhpParser/Node/Expr/YieldFrom.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace PhpParser\Node\Expr;
+
+use PhpParser\Node\Expr;
+
+class YieldFrom extends Expr
+{
+    /** @var Expr Expression to yield from */
+    public $expr;
+
+    /**
+     * Constructs an "yield from" node.
+     *
+     * @param Expr  $expr       Expression
+     * @param array $attributes Additional attributes
+     */
+    public function __construct(Expr $expr, array $attributes = array()) {
+        parent::__construct(null, $attributes);
+        $this->expr = $expr;
+    }
+
+    public function getSubNodeNames() {
+        return array('expr');
+    }
+}

lib/PhpParser/Node/FunctionLike.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace PhpParser\Node;
+
+use PhpParser\Node;
+
+interface FunctionLike extends Node
+{
+    /**
+     * Whether to return by reference
+     *
+     * @return bool
+     */
+    public function returnsByRef();
+
+    /**
+     * List of parameters
+     *
+     * @return Node\Param[]
+     */
+    public function getParams();
+
+    /**
+     * Get the declared return type or null
+     * 
+     * @return null|string|Node\Name
+     */
+    public function getReturnType();
+
+    /**
+     * The function body
+     *
+     * @return Node\Stmt[]
+     */
+    public function getStmts();
+}

lib/PhpParser/Node/Param.php

@@ -37,7 +37,7 @@ class Param extends NodeAbstract
         $this->default = $default;
 
         if ($variadic && null !== $default) {
-            throw new Error('Variadic parameter cannot have a default value');
+            throw new Error('Variadic parameter cannot have a default value', $default->getAttributes());
         }
     }
 

lib/PhpParser/Node/Stmt/ClassLike.php

@@ -10,6 +10,11 @@ abstract class ClassLike extends Node\Stmt {
     /** @var Node[] Statements */
     public $stmts;
 
+    /**
+     * Gets all methods defined directly in this class/interface/trait
+     *
+     * @return ClassMethod[]
+     */
     public function getMethods() {
         $methods = array();
         foreach ($this->stmts as $stmt) {
@@ -19,4 +24,21 @@ abstract class ClassLike extends Node\Stmt {
         }
         return $methods;
     }
+
+    /**
+     * Gets method with the given name defined directly in this class/interface/trait.
+     *
+     * @param string $name Name of the method (compared case-insensitively)
+     *
+     * @return ClassMethod|null Method node or null if the method does not exist
+     */
+    public function getMethod($name) {
+        $lowerName = strtolower($name);
+        foreach ($this->stmts as $stmt) {
+            if ($stmt instanceof ClassMethod && $lowerName === strtolower($stmt->name)) {
+                return $stmt;
+            }
+        }
+        return null;
+    }
 }

lib/PhpParser/Node/Stmt/ClassMethod.php

@@ -3,9 +3,10 @@
 namespace PhpParser\Node\Stmt;
 
 use PhpParser\Node;
+use PhpParser\Node\FunctionLike;
 use PhpParser\Error;
 
-class ClassMethod extends Node\Stmt
+class ClassMethod extends Node\Stmt implements FunctionLike
 {
     /** @var int Type */
     public $type;
@@ -15,7 +16,7 @@ class ClassMethod extends Node\Stmt
     public $name;
     /** @var Node\Param[] Parameters */
     public $params;
-    /** @var null|string|Node\Name[] Return type */
+    /** @var null|string|Node\Name Return type */
     public $returnType;
     /** @var Node[] Statements */
     public $stmts;
@@ -57,8 +58,25 @@ class ClassMethod extends Node\Stmt
         return array('type', 'byRef', 'name', 'params', 'returnType', 'stmts');
     }
 
+    public function returnsByRef() {
+        return $this->byRef;
+    }
+
+    public function getParams() {
+        return $this->params;
+    }
+
+    public function getReturnType() {
+        return $this->returnType;
+    }
+
+    public function getStmts() {
+        return $this->stmts;
+    }
+
     public function isPublic() {
-        return ($this->type & Class_::MODIFIER_PUBLIC) !== 0 || $this->type === 0;
+        return ($this->type & Class_::MODIFIER_PUBLIC) !== 0
+            || ($this->type & Class_::VISIBILITY_MODIFER_MASK) === 0;
     }
 
     public function isProtected() {

lib/PhpParser/Node/Stmt/Class_.php

@@ -32,7 +32,7 @@ class Class_ extends ClassLike
     /**
      * Constructs a class node.
      *
-     * @param string      $name       Name
+     * @param string|null $name       Name
      * @param array       $subNodes   Array of the following optional subnodes:
      *                                'type'       => 0      : Type
      *                                'extends'    => null   : Name of extended class
@@ -48,17 +48,23 @@ class Class_ extends ClassLike
         $this->implements = isset($subNodes['implements']) ? $subNodes['implements'] : array();
         $this->stmts = isset($subNodes['stmts']) ? $subNodes['stmts'] : array();
 
-        if (isset(self::$specialNames[(string) $this->name])) {
+        if (null !== $this->name && isset(self::$specialNames[strtolower($this->name)])) {
             throw new Error(sprintf('Cannot use \'%s\' as class name as it is reserved', $this->name));
         }
 
-        if (isset(self::$specialNames[(string) $this->extends])) {
-            throw new Error(sprintf('Cannot use \'%s\' as class name as it is reserved', $this->extends));
+        if (isset(self::$specialNames[strtolower($this->extends)])) {
+            throw new Error(
+                sprintf('Cannot use \'%s\' as class name as it is reserved', $this->extends),
+                $this->extends->getAttributes()
+            );
         }
 
         foreach ($this->implements as $interface) {
-            if (isset(self::$specialNames[(string) $interface])) {
-                throw new Error(sprintf('Cannot use \'%s\' as interface name as it is reserved', $interface));
+            if (isset(self::$specialNames[strtolower($interface)])) {
+                throw new Error(
+                    sprintf('Cannot use \'%s\' as interface name as it is reserved', $interface),
+                    $interface->getAttributes()
+                );
             }
         }
     }
@@ -75,6 +81,10 @@ class Class_ extends ClassLike
         return (bool) ($this->type & self::MODIFIER_FINAL);
     }
 
+    public function isAnonymous() {
+        return null === $this->name;
+    }
+
     /**
      * @internal
      */

lib/PhpParser/Node/Stmt/Function_.php

@@ -3,8 +3,9 @@
 namespace PhpParser\Node\Stmt;
 
 use PhpParser\Node;
+use PhpParser\Node\FunctionLike;
 
-class Function_ extends Node\Stmt
+class Function_ extends Node\Stmt implements FunctionLike
 {
     /** @var bool Whether function returns by reference */
     public $byRef;
@@ -12,7 +13,7 @@ class Function_ extends Node\Stmt
     public $name;
     /** @var Node\Param[] Parameters */
     public $params;
-    /** @var null|string|Node\Name[] Return type */
+    /** @var null|string|Node\Name Return type */
     public $returnType;
     /** @var Node[] Statements */
     public $stmts;
@@ -40,4 +41,20 @@ class Function_ extends Node\Stmt
     public function getSubNodeNames() {
         return array('byRef', 'name', 'params', 'returnType', 'stmts');
     }
+
+    public function returnsByRef() {
+        return $this->byRef;
+    }
+
+    public function getParams() {
+        return $this->params;
+    }
+
+    public function getReturnType() {
+        return $this->returnType;
+    }
+
+    public function getStmts() {
+        return $this->stmts;
+    }
 }

lib/PhpParser/Node/Stmt/Interface_.php

@@ -31,13 +31,16 @@ class Interface_ extends ClassLike
         $this->extends = isset($subNodes['extends']) ? $subNodes['extends'] : array();
         $this->stmts = isset($subNodes['stmts']) ? $subNodes['stmts'] : array();
 
-        if (isset(self::$specialNames[(string) $this->name])) {
+        if (isset(self::$specialNames[strtolower($this->name)])) {
             throw new Error(sprintf('Cannot use \'%s\' as class name as it is reserved', $this->name));
         }
 
         foreach ($this->extends as $interface) {
-            if (isset(self::$specialNames[(string) $interface])) {
-                throw new Error(sprintf('Cannot use \'%s\' as interface name as it is reserved', $interface));
+            if (isset(self::$specialNames[strtolower($interface)])) {
+                throw new Error(
+                    sprintf('Cannot use \'%s\' as interface name as it is reserved', $interface),
+                    $interface->getAttributes()
+                );
             }
         }
     }

lib/PhpParser/Node/Stmt/Namespace_.php

@@ -30,14 +30,17 @@ class Namespace_ extends Node\Stmt
         $this->name = $name;
         $this->stmts = $stmts;
 
-        if (isset(self::$specialNames[(string) $this->name])) {
-            throw new Error(sprintf('Cannot use \'%s\' as namespace name', $this->name));
+        if (isset(self::$specialNames[strtolower($this->name)])) {
+            throw new Error(
+                sprintf('Cannot use \'%s\' as namespace name', $this->name),
+                $this->name->getAttributes()
+            );
         }
 
         if (null !== $this->stmts) {
             foreach ($this->stmts as $stmt) {
                 if ($stmt instanceof self) {
-                    throw new Error('Namespace declarations cannot be nested', $stmt->getLine());
+                    throw new Error('Namespace declarations cannot be nested', $stmt->getAttributes());
                 }
             }
         }

lib/PhpParser/Node/Stmt/Property.php

@@ -38,7 +38,8 @@ class Property extends Node\Stmt
     }
 
     public function isPublic() {
-        return ($this->type & Class_::MODIFIER_PUBLIC) !== 0 || $this->type === 0;
+        return ($this->type & Class_::MODIFIER_PUBLIC) !== 0
+            || ($this->type & Class_::VISIBILITY_MODIFER_MASK) === 0;
     }
 
     public function isProtected() {

lib/PhpParser/Node/Stmt/UseUse.php

@@ -24,7 +24,7 @@ class UseUse extends Node\Stmt
             $alias = $name->getLast();
         }
 
-        if ('self' == $alias || 'parent' == $alias) {
+        if ('self' == strtolower($alias) || 'parent' == strtolower($alias)) {
             throw new Error(sprintf(
                 'Cannot use %s as %s because \'%2$s\' is a special class name',
                 $name, $alias

lib/PhpParser/NodeVisitor/NameResolver.php

@@ -37,7 +37,9 @@ class NameResolver extends NodeVisitorAbstract
                 $interface = $this->resolveClassName($interface);
             }
 
+            if (null !== $node->name) {
                 $this->addNamespacedName($node);
+            }
         } elseif ($node instanceof Stmt\Interface_) {
             foreach ($node->extends as &$interface) {
                 $interface = $this->resolveClassName($interface);

lib/PhpParser/ParserAbstract.php

@@ -23,6 +23,8 @@ abstract class ParserAbstract
 
     /** @var int Symbol number signifying an invalid token */
     protected $invalidSymbol;
+    /** @var int Symbol number of error recovery token */
+    protected $errorSymbol;
     /** @var int Action number signifying default action */
     protected $defaultAction;
     /** @var int Rule number signifying that an unexpected token was encountered */
@@ -76,18 +78,43 @@ abstract class ParserAbstract
     protected $lexer;
     /** @var mixed Temporary value containing the result of last semantic action (reduction) */
     protected $semValue;
-    /** @var array Semantic value stack (contains values of tokens and semantic action results) */
-    protected $semStack;
     /** @var int Position in stacks (state stack, semantic value stack, attribute stack) */
     protected $stackPos;
+    /** @var array Semantic value stack (contains values of tokens and semantic action results) */
+    protected $semStack;
+    /** @var array[] Start attribute stack */
+    protected $startAttributeStack;
+    /** @var array End attributes of last *shifted* token */
+    protected $endAttributes;
+
+    /** @var bool Whether to throw on first error */
+    protected $throwOnError;
+    /** @var Error[] Errors collected during last parse */
+    protected $errors;
 
     /**
      * Creates a parser instance.
      *
      * @param Lexer $lexer A lexer
+     * @param array $options Options array. The boolean 'throwOnError' option determines whether an exception should be
+     *                       thrown on first error, or if the parser should try to continue parsing the remaining code
+     *                       and build a partial AST.
      */
-    public function __construct(Lexer $lexer) {
+    public function __construct(Lexer $lexer, array $options = array()) {
         $this->lexer = $lexer;
+        $this->errors = array();
+        $this->throwOnError = isset($options['throwOnError']) ? $options['throwOnError'] : true;
+    }
+
+    /**
+     * Get array of errors that occurred during the last parse.
+     *
+     * This method may only return multiple errors if the 'throwOnError' option is disabled.
+     *
+     * @return Error[]
+     */
+    public function getErrors() {
+        return $this->errors;
     }
 
     /**
@@ -95,10 +122,12 @@ abstract class ParserAbstract
      *
      * @param string $code The source code to parse
      *
-     * @return Node[] Array of statements
+     * @return Node[]|null Array of statements (or null if the 'throwOnError' option is disabled and the parser was
+     *                     unable to recover from an error).
      */
     public function parse($code) {
         $this->lexer->startLexing($code);
+        $this->errors = array();
 
         // We start off with no lookahead-token
         $symbol = self::SYMBOL_NONE;
@@ -106,12 +135,13 @@ abstract class ParserAbstract
         // The attributes for a node are taken from the first and last token of the node.
         // From the first token only the startAttributes are taken and from the last only
         // the endAttributes. Both are merged using the array union operator (+).
-        $startAttributes = array('startLine' => 1);
-        $endAttributes   = array();
+        $startAttributes = '*POISON';
+        $endAttributes = '*POISON';
+        $this->endAttributes = $endAttributes;
 
         // In order to figure out the attributes for the starting token, we have to keep
         // them in a stack
-        $attributeStack = array($startAttributes);
+        $this->startAttributeStack = array();
 
         // Start off in the initial state and keep a stack of previous states
         $state = 0;
@@ -123,6 +153,8 @@ abstract class ParserAbstract
         // Current position in the stack(s)
         $this->stackPos = 0;
 
+        $errorState = 0;
+
         for (;;) {
             //$this->traceNewState($state, $symbol);
 
@@ -134,7 +166,7 @@ abstract class ParserAbstract
                     // The end attributes are fetched into a temporary variable and only set once the token is really
                     // shifted (not during read). Otherwise you would sometimes get off-by-one errors, when a rule is
                     // reduced after a token was read but not yet shifted.
-                    $tokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $nextEndAttributes);
+                    $tokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $endAttributes);
 
                     // map the lexer token id to the internally used symbols
                     $symbol = $tokenId >= 0 && $tokenId < $this->tokenToSymbolMapSize
@@ -148,7 +180,9 @@ abstract class ParserAbstract
                         ));
                     }
 
-                    $attributeStack[$this->stackPos] = $startAttributes;
+                    // This is necessary to assign some meaningful attributes to /* empty */ productions. They'll get
+                    // the attributes of the next token, even though they don't contain it themselves.
+                    $this->startAttributeStack[$this->stackPos+1] = $startAttributes;
 
                     //$this->traceRead($symbol);
                 }
@@ -173,12 +207,17 @@ abstract class ParserAbstract
                         ++$this->stackPos;
                         $stateStack[$this->stackPos] = $state = $action;
                         $this->semStack[$this->stackPos] = $tokenValue;
-                        $attributeStack[$this->stackPos] = $startAttributes;
-                        $endAttributes = $nextEndAttributes;
+                        $this->startAttributeStack[$this->stackPos] = $startAttributes;
+                        $this->endAttributes = $endAttributes;
                         $symbol = self::SYMBOL_NONE;
 
-                        if ($action < $this->YYNLSTATES)
+                        if ($errorState) {
+                            --$errorState;
+                        }
+
+                        if ($action < $this->YYNLSTATES) {
                             continue;
+                        }
 
                         /* $yyn >= YYNLSTATES means shift-and-reduce */
                         $rule = $action - $this->YYNLSTATES;
@@ -200,16 +239,19 @@ abstract class ParserAbstract
                     //$this->traceReduce($rule);
 
                     try {
-                        $this->{'reduceRule' . $rule}(
-                            $attributeStack[$this->stackPos - $this->ruleToLength[$rule]]
-                            + $endAttributes
-                        );
+                        $this->{'reduceRule' . $rule}();
                     } catch (Error $e) {
-                        if (-1 === $e->getRawLine() && isset($startAttributes['startLine'])) {
-                            $e->setRawLine($startAttributes['startLine']);
+                        if (-1 === $e->getStartLine() && isset($startAttributes['startLine'])) {
+                            $e->setStartLine($startAttributes['startLine']);
                         }
 
+                        $this->errors[] = $e;
+                        if ($this->throwOnError) {
                             throw $e;
+                        } else {
+                            // Currently can't recover from "special" errors
+                            return null;
+                        }
                     }
 
                     /* Goto - shift nonterminal */
@@ -225,27 +267,72 @@ abstract class ParserAbstract
                     ++$this->stackPos;
                     $stateStack[$this->stackPos]     = $state;
                     $this->semStack[$this->stackPos] = $this->semValue;
-                    $attributeStack[$this->stackPos] = $startAttributes;
-
-                    if ($state < $this->YYNLSTATES)
-                        break;
-                    /* >= YYNLSTATES means shift-and-reduce */
-                    $rule = $state - $this->YYNLSTATES;
                 } else {
                     /* error */
-                    if ($expected = $this->getExpectedTokens($state)) {
-                        $expectedString = ', expecting ' . implode(' or ', $expected);
-                    } else {
-                        $expectedString = '';
+                    switch ($errorState) {
+                        case 0:
+                            $msg = $this->getErrorMessage($symbol, $state);
+                            $error = new Error($msg, $startAttributes + $endAttributes);
+                            $this->errors[] = $error;
+                            if ($this->throwOnError) {
+                                throw $error;
+                            }
+                            // Break missing intentionally
+                        case 1:
+                        case 2:
+                            $errorState = 3;
+
+                            // Pop until error-expecting state uncovered
+                            while (!(
+                                (($idx = $this->actionBase[$state] + $this->errorSymbol) >= 0
+                                    && $idx < $this->actionTableSize && $this->actionCheck[$idx] == $this->errorSymbol)
+                                || ($state < $this->YY2TBLSTATE
+                                    && ($idx = $this->actionBase[$state + $this->YYNLSTATES] + $this->errorSymbol) >= 0
+                                    && $idx < $this->actionTableSize && $this->actionCheck[$idx] == $this->errorSymbol)
+                            ) || ($action = $this->action[$idx]) == $this->defaultAction) { // Not totally sure about this
+                                if ($this->stackPos <= 0) {
+                                    // Could not recover from error
+                                    return null;
+                                }
+                                $state = $stateStack[--$this->stackPos];
+                                //$this->tracePop($state);
                             }
 
-                    throw new Error(
-                        'Syntax error, unexpected ' . $this->symbolToName[$symbol] . $expectedString,
-                        $startAttributes['startLine']
-                    );
+                            //$this->traceShift($this->errorSymbol);
+                            $stateStack[++$this->stackPos] = $state = $action;
+                            break;
+
+                        case 3:
+                            if ($symbol === 0) {
+                                // Reached EOF without recovering from error
+                                return null;
+                            }
+
+                            //$this->traceDiscard($symbol);
+                            $symbol = self::SYMBOL_NONE;
+                            break 2;
                     }
                 }
+
+                if ($state < $this->YYNLSTATES) {
+                    break;
                 }
+
+                /* >= YYNLSTATES means shift-and-reduce */
+                $rule = $state - $this->YYNLSTATES;
+            }
+        }
+
+        throw new \RuntimeException('Reached end of parser loop');
+    }
+
+    protected function getErrorMessage($symbol, $state) {
+        $expectedString = '';
+        if ($expected = $this->getExpectedTokens($state)) {
+            $expectedString = ', expecting ' . implode(' or ', $expected);
+        }
+
+        return 'Syntax error, unexpected ' . $this->symbolToName[$symbol] . $expectedString;
     }
 
     protected function getExpectedTokens($state) {
@@ -277,6 +364,7 @@ abstract class ParserAbstract
      * Tracing functions used for debugging the parser.
      */
 
+    /*
     protected function traceNewState($state, $symbol) {
         echo '% State ' . $state
             . ', Lookahead ' . ($symbol == self::SYMBOL_NONE ? '--none--' : $this->symbolToName[$symbol]) . "\n";
@@ -298,6 +386,15 @@ abstract class ParserAbstract
         echo '% Reduce by (' . $n . ') ' . $this->productions[$n] . "\n";
     }
 
+    protected function tracePop($state) {
+        echo '% Recovering, uncovered state ' . $state . "\n";
+    }
+
+    protected function traceDiscard($symbol) {
+        echo '% Discard ' . $this->symbolToName[$symbol] . "\n";
+    }
+    */
+
     /*
      * Helper functions invoked by semantic actions
      */

lib/PhpParser/PrettyPrinter/Standard.php

@@ -96,7 +96,10 @@ class Standard extends PrettyPrinterAbstract
     }
 
     public function pScalar_DNumber(Scalar\DNumber $node) {
-        $stringValue = (string) $node->value;
+        $stringValue = sprintf('%.16G', $node->value);
+        if ($node->value !== (double) $stringValue) {
+            $stringValue = sprintf('%.17G', $node->value);
+        }
 
         // ensure that number is really printed as float
         return preg_match('/^-?[0-9]+$/', $stringValue) ? $stringValue . '.0' : $stringValue;
@@ -312,6 +315,14 @@ class Standard extends PrettyPrinterAbstract
         return $this->pPrefixOp('Expr_ErrorSuppress', '@', $node->expr);
     }
 
+    public function pExpr_YieldFrom(Expr\YieldFrom $node) {
+        return $this->pPrefixOp('Expr_YieldFrom', 'yield from ', $node->expr);
+    }
+
+    public function pExpr_Print(Expr\Print_ $node) {
+        return $this->pPrefixOp('Expr_Print', 'print ', $node->expr);
+    }
+
     // Casts
 
     public function pExpr_Cast_Int(Cast\Int_ $node) {
@@ -372,10 +383,6 @@ class Standard extends PrettyPrinterAbstract
         return 'isset(' . $this->pCommaSeparated($node->vars) . ')';
     }
 
-    public function pExpr_Print(Expr\Print_ $node) {
-        return 'print ' . $this->p($node->expr);
-    }
-
     public function pExpr_Eval(Expr\Eval_ $node) {
         return 'eval(' . $this->p($node->expr) . ')';
     }
@@ -462,6 +469,10 @@ class Standard extends PrettyPrinterAbstract
     }
 
     public function pExpr_New(Expr\New_ $node) {
+        if ($node->class instanceof Stmt\Class_) {
+            $args = $node->args ? '(' . $this->pCommaSeparated($node->args) . ')' : '';
+            return 'new ' . $this->pClassCommon($node->class, $args);
+        }
         return 'new ' . $this->p($node->class) . '(' . $this->pCommaSeparated($node->args) . ')';
     }
 
@@ -523,11 +534,7 @@ class Standard extends PrettyPrinterAbstract
     }
 
     public function pStmt_Class(Stmt\Class_ $node) {
-        return $this->pModifiers($node->type)
-             . 'class ' . $node->name
-             . (null !== $node->extends ? ' extends ' . $this->p($node->extends) : '')
-             . (!empty($node->implements) ? ' implements ' . $this->pCommaSeparated($node->implements) : '')
-             . "\n" . '{' . $this->pStmts($node->stmts) . "\n" . '}';
+        return $this->pClassCommon($node, ' ' . $node->name);
     }
 
     public function pStmt_Trait(Stmt\Trait_ $node) {
@@ -725,6 +732,14 @@ class Standard extends PrettyPrinterAbstract
         return is_string($node) ? $node : $this->p($node);
     }
 
+    protected function pClassCommon(Stmt\Class_ $node, $afterClassToken) {
+        return $this->pModifiers($node->type)
+        . 'class' . $afterClassToken
+        . (null !== $node->extends ? ' extends ' . $this->p($node->extends) : '')
+        . (!empty($node->implements) ? ' implements ' . $this->pCommaSeparated($node->implements) : '')
+        . "\n" . '{' . $this->pStmts($node->stmts) . "\n" . '}';
+    }
+
     /** @internal */
     public function pObjectProperty($node) {
         if ($node instanceof Expr) {

lib/PhpParser/PrettyPrinterAbstract.php

@@ -66,6 +66,8 @@ abstract class PrettyPrinterAbstract
         'Expr_AssignOp_ShiftLeft'      => array(160,  1),
         'Expr_AssignOp_ShiftRight'     => array(160,  1),
         'Expr_AssignOp_Pow'            => array(160,  1),
+        'Expr_YieldFrom'               => array(165,  1),
+        'Expr_Print'                   => array(168,  1),
         'Expr_BinaryOp_LogicalAnd'     => array(170, -1),
         'Expr_BinaryOp_LogicalXor'     => array(180, -1),
         'Expr_BinaryOp_LogicalOr'      => array(190, -1),

test/PhpParser/Builder/TraitTest.php

@@ -17,13 +17,17 @@ class TraitTest extends \PHPUnit_Framework_TestCase
         $method1 = new Stmt\ClassMethod('test1');
         $method2 = new Stmt\ClassMethod('test2');
         $method3 = new Stmt\ClassMethod('test3');
+        $prop = new Stmt\Property(Stmt\Class_::MODIFIER_PUBLIC, array(
+            new Stmt\PropertyProperty('test')
+        ));
         $trait = $this->createTraitBuilder('TestTrait')
             ->setDocComment('/** Nice trait */')
             ->addStmt($method1)
             ->addStmts(array($method2, $method3))
+            ->addStmt($prop)
             ->getNode();
         $this->assertEquals(new Stmt\Trait_('TestTrait', array(
-            $method1, $method2, $method3
+            $prop, $method1, $method2, $method3
         ), array(
             'comments' => array(
                 new Comment\Doc('/** Nice trait */')

test/PhpParser/Builder/UseTest.php

@@ -26,4 +26,10 @@ class UseTest extends \PHPUnit_Framework_TestCase
             new Stmt\UseUse(new Name('foo\bar'), 'foo')
         ), Stmt\Use_::TYPE_FUNCTION), $node);
     }
+
+    public function testNonExistingMethod() {
+        $this->setExpectedException('LogicException', 'Method "foo" does not exist');
+        $builder = $this->createUseBuilder('Test');
+        $builder->foo();
+    }
 }

test/PhpParser/BuilderFactoryTest.php

@@ -28,6 +28,12 @@ class BuilderFactoryTest extends \PHPUnit_Framework_TestCase
         );
     }
 
+    public function testNonExistingMethod() {
+        $this->setExpectedException('LogicException', 'Method "foo" does not exist');
+        $factory = new BuilderFactory();
+        $factory->foo();
+    }
+
     public function testIntegration() {
         $factory = new BuilderFactory;
         $node = $factory->namespace('Name\Space')

test/PhpParser/ErrorTest.php

@@ -5,9 +5,16 @@ namespace PhpParser;
 class ErrorTest extends \PHPUnit_Framework_TestCase
 {
     public function testConstruct() {
-        $error = new Error('Some error', 10);
+        $attributes = array(
+            'startLine' => 10,
+            'endLine' => 11,
+        );
+        $error = new Error('Some error', $attributes);
 
         $this->assertSame('Some error', $error->getRawMessage());
+        $this->assertSame($attributes, $error->getAttributes());
+        $this->assertSame(10, $error->getStartLine());
+        $this->assertSame(11, $error->getEndLine());
         $this->assertSame(10, $error->getRawLine());
         $this->assertSame('Some error on line 10', $error->getMessage());
 
@@ -19,17 +26,87 @@ class ErrorTest extends \PHPUnit_Framework_TestCase
      */
     public function testSetMessageAndLine(Error $error) {
         $error->setRawMessage('Some other error');
-        $error->setRawLine(15);
-
         $this->assertSame('Some other error', $error->getRawMessage());
-        $this->assertSame(15, $error->getRawLine());
+
+        $error->setStartLine(15);
+        $this->assertSame(15, $error->getStartLine());
         $this->assertSame('Some other error on line 15', $error->getMessage());
+
+        $error->setRawLine(17);
+        $this->assertSame(17, $error->getRawLine());
+        $this->assertSame('Some other error on line 17', $error->getMessage());
     }
 
     public function testUnknownLine() {
         $error = new Error('Some error');
 
+        $this->assertSame(-1, $error->getStartLine());
+        $this->assertSame(-1, $error->getEndLine());
         $this->assertSame(-1, $error->getRawLine());
         $this->assertSame('Some error on unknown line', $error->getMessage());
     }
+
+    /** @dataProvider provideTestColumnInfo */
+    public function testColumnInfo($code, $startPos, $endPos, $startColumn, $endColumn) {
+        $error = new Error('Some error', array(
+            'startFilePos' => $startPos,
+            'endFilePos' => $endPos,
+        ));
+
+        $this->assertSame(true, $error->hasColumnInfo());
+        $this->assertSame($startColumn, $error->getStartColumn($code));
+        $this->assertSame($endColumn, $error->getEndColumn($code));
+
+    }
+
+    public function provideTestColumnInfo() {
+        return array(
+            // Error at "bar"
+            array("<?php foo bar baz", 10, 12, 11, 13),
+            array("<?php\nfoo bar baz", 10, 12, 5, 7),
+            array("<?php foo\nbar baz", 10, 12, 1, 3),
+            array("<?php foo bar\nbaz", 10, 12, 11, 13),
+            array("<?php\r\nfoo bar baz", 11, 13, 5, 7),
+            // Error at "baz"
+            array("<?php foo bar baz", 14, 16, 15, 17),
+            array("<?php foo bar\nbaz", 14, 16, 1, 3),
+            // Error at string literal
+            array("<?php foo 'bar\nbaz' xyz", 10, 18, 11, 4),
+            array("<?php\nfoo 'bar\nbaz' xyz", 10, 18, 5, 4),
+            array("<?php foo\n'\nbarbaz\n'\nxyz", 10, 19, 1, 1),
+            // Error over full string
+            array("<?php", 0, 4, 1, 5),
+            array("<?\nphp", 0, 5, 1, 3),
+        );
+    }
+
+    public function testNoColumnInfo() {
+        $error = new Error('Some error', 3);
+
+        $this->assertSame(false, $error->hasColumnInfo());
+        try {
+            $error->getStartColumn('');
+            $this->fail('Expected RuntimeException');
+        } catch (\RuntimeException $e) {
+            $this->assertSame('Error does not have column information', $e->getMessage());
+        }
+        try {
+            $error->getEndColumn('');
+            $this->fail('Expected RuntimeException');
+        } catch (\RuntimeException $e) {
+            $this->assertSame('Error does not have column information', $e->getMessage());
+        }
+    }
+
+    /**
+     * @expectedException \RuntimeException
+     * @expectedExceptionMessage Invalid position information
+     */
+    public function testInvalidPosInfo() {
+        $error = new Error('Some error', array(
+            'startFilePos' => 10,
+            'endFilePos' => 11,
+        ));
+        $error->getStartColumn('code');
+    }
 }

test/PhpParser/Lexer/EmulativeTest.php

@@ -87,6 +87,12 @@ class EmulativeTest extends LexerTest
 
     public function provideTestLexNewFeatures() {
         return array(
+            array('yield from', array(
+                array(Parser::T_YIELD_FROM, 'yield from'),
+            )),
+            array("yield\r\nfrom", array(
+                array(Parser::T_YIELD_FROM, "yield\r\nfrom"),
+            )),
             array('...', array(
                 array(Parser::T_ELLIPSIS, '...'),
             )),

test/PhpParser/LexerTest.php

@@ -215,6 +215,18 @@ class LexerTest extends \PHPUnit_Framework_TestCase
         );
     }
 
+    /**
+     * @expectedException \PhpParser\Error
+     * @expectedExceptionMessage __HALT_COMPILER must be followed by "();"
+     */
+    public function testHandleHaltCompilerError() {
+        $lexer = $this->getLexer();
+        $lexer->startLexing('<?php ... __halt_compiler invalid ();');
+
+        while (Parser::T_HALT_COMPILER !== $lexer->getNextToken());
+        $lexer->handleHaltCompiler();
+    }
+
     public function testGetTokens() {
         $code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
         $expectedTokens = array(

test/PhpParser/Node/Stmt/ClassMethodTest.php

@@ -36,4 +36,28 @@ class ClassMethodTest extends \PHPUnit_Framework_TestCase
             array('static'),
         );
     }
+
+    /**
+     * Checks that implicit public modifier detection for method is working
+     *
+     * @dataProvider implicitPublicModifiers
+     *
+     * @param integer $modifier Node type modifier
+     */
+    public function testImplicitPublic($modifier)
+    {
+        $node = new ClassMethod('foo', array(
+            'type' => constant('PhpParser\Node\Stmt\Class_::MODIFIER_' . strtoupper($modifier))
+        ));
+
+        $this->assertTrue($node->isPublic(), 'Node should be implicitly public');
+    }
+
+    public function implicitPublicModifiers() {
+        return array(
+            array('abstract'),
+            array('final'),
+            array('static'),
+        );
+    }
 }

test/PhpParser/Node/Stmt/ClassTest.php

@@ -39,4 +39,21 @@ class ClassTest extends \PHPUnit_Framework_TestCase
 
         $this->assertSame($methods, $class->getMethods());
     }
+
+    public function testGetMethod() {
+        $methodConstruct = new ClassMethod('__CONSTRUCT');
+        $methodTest = new ClassMethod('test');
+        $class = new Class_('Foo', array(
+            'stmts' => array(
+                new ClassConst(array()),
+                $methodConstruct,
+                new Property(0, array()),
+                $methodTest,
+            )
+        ));
+
+        $this->assertSame($methodConstruct, $class->getMethod('__construct'));
+        $this->assertSame($methodTest, $class->getMethod('test'));
+        $this->assertNull($class->getMethod('nonExisting'));
+    }
 }

test/PhpParser/Node/Stmt/PropertyTest.php

@@ -25,6 +25,14 @@ class PropertyTest extends \PHPUnit_Framework_TestCase
         $this->assertFalse($node->isStatic());
     }
 
+    public function testStaticImplicitlyPublic() {
+        $node = new Property(Class_::MODIFIER_STATIC, array());
+        $this->assertTrue($node->isPublic());
+        $this->assertFalse($node->isProtected());
+        $this->assertFalse($node->isPrivate());
+        $this->assertTrue($node->isStatic());
+    }
+
     public function provideModifiers() {
         return array(
             array('public'),

test/PhpParser/NodeVisitor/NameResolverTest.php

@@ -257,6 +257,7 @@ EOC;
             new Stmt\Const_(array(
                 new Node\Const_('D', new Node\Scalar\String_('E'))
             )),
+            new Expr\New_(new Stmt\Class_(null)),
         ));
 
         $traverser = new PhpParser\NodeTraverser;
@@ -268,10 +269,12 @@ EOC;
         $this->assertSame('NS\\B', (string) $stmts[0]->stmts[1]->namespacedName);
         $this->assertSame('NS\\C', (string) $stmts[0]->stmts[2]->namespacedName);
         $this->assertSame('NS\\D', (string) $stmts[0]->stmts[3]->consts[0]->namespacedName);
+        $this->assertObjectNotHasAttribute('namespacedName', $stmts[0]->stmts[4]->class);
         $this->assertSame('A',     (string) $stmts[1]->stmts[0]->namespacedName);
         $this->assertSame('B',     (string) $stmts[1]->stmts[1]->namespacedName);
         $this->assertSame('C',     (string) $stmts[1]->stmts[2]->namespacedName);
         $this->assertSame('D',     (string) $stmts[1]->stmts[3]->consts[0]->namespacedName);
+        $this->assertObjectNotHasAttribute('namespacedName', $stmts[1]->stmts[4]->class);
     }
 
     public function testAddTraitNamespacedName() {

test/PhpParser/ParserTest.php

@@ -11,39 +11,59 @@ class ParserTest extends CodeTestAbstract
     /**
      * @dataProvider provideTestParse
      */
-    public function testParse($name, $code, $dump) {
-        $parser = new Parser(new Lexer\Emulative);
-        $dumper = new NodeDumper;
+    public function testParse($name, $code, $expected) {
+        $lexer = new Lexer\Emulative(array('usedAttributes' => array(
+            'startLine', 'endLine', 'startFilePos', 'endFilePos'
+        )));
+        $parser = new Parser($lexer, array(
+            'throwOnError' => false,
+        ));
 
         $stmts = $parser->parse($code);
-        $this->assertSame(
-            $this->canonicalize($dump),
-            $this->canonicalize($dumper->dump($stmts)),
-            $name
-        );
+        $errors = $parser->getErrors();
+
+        $output = '';
+        foreach ($errors as $error) {
+            $output .= $this->formatErrorMessage($error, $code) . "\n";
+        }
+
+        if (null !== $stmts) {
+            $dumper = new NodeDumper;
+            $output .= $dumper->dump($stmts);
+        }
+
+        $this->assertSame($this->canonicalize($expected), $this->canonicalize($output), $name);
     }
 
     public function provideTestParse() {
         return $this->getTests(__DIR__ . '/../code/parser', 'test');
     }
 
+    private function formatErrorMessage(Error $e, $code) {
+        if ($e->hasColumnInfo()) {
+            return $e->getRawMessage() . ' from ' . $e->getStartLine() . ':' . $e->getStartColumn($code)
+                . ' to ' . $e->getEndLine() . ':' . $e->getEndColumn($code);
+        } else {
+            return $e->getMessage();
+        }
+    }
+
     /**
-     * @dataProvider provideTestParseFail
+     * @expectedException \PhpParser\Error
+     * @expectedExceptionMessage Syntax error, unexpected EOF on line 1
      */
-    public function testParseFail($name, $code, $msg) {
-        $parser = new Parser(new Lexer\Emulative);
-
-        try {
-            $parser->parse($code);
-
-            $this->fail(sprintf('"%s": Expected Error', $name));
-        } catch (Error $e) {
-            $this->assertSame($msg, $e->getMessage(), $name);
-        }
+    public function testParserThrowsSyntaxError() {
+        $parser = new Parser(new Lexer());
+        $parser->parse('<?php foo');
     }
 
-    public function provideTestParseFail() {
-        return $this->getTests(__DIR__ . '/../code/parser', 'test-fail');
+    /**
+     * @expectedException \PhpParser\Error
+     * @expectedExceptionMessage Cannot use foo as self because 'self' is a special class name on line 1
+     */
+    public function testParserThrowsSpecialError() {
+        $parser = new Parser(new Lexer());
+        $parser->parse('<?php use foo as self;');
     }
 
     public function testAttributeAssignment() {
@@ -124,19 +144,6 @@ EOC;
         $parser = new Parser($lexer);
         $parser->parse('dummy');
     }
-
-    public function testInvalidOctals() {
-        if (version_compare(PHP_VERSION, '7.0-dev', '>=')) {
-            $this->markTestSkipped('Cannot parse invalid octal numbers on PHP 7');
-        }
-
-        $parser = new Parser(new Lexer);
-        $stmts = $parser->parse('<?php 0787; 0177777777777777777777787;');
-        $this->assertInstanceof('PhpParser\Node\Scalar\LNumber', $stmts[0]);
-        $this->assertInstanceof('PhpParser\Node\Scalar\DNumber', $stmts[1]);
-        $this->assertSame(7, $stmts[0]->value);
-        $this->assertSame(0xFFFFFFFFFFFFFFFF, $stmts[1]->value);
-    }
 }
 
 class InvalidTokenLexer extends Lexer {

test/code/parser/errorHandling/eofError.test

@@ -0,0 +1,13 @@
+Error positions
+-----
+<?php foo
+-----
+Syntax error, unexpected EOF from 1:10 to 1:10
+array(
+)
+-----
+<?php foo /* bar */
+-----
+Syntax error, unexpected EOF from 1:20 to 1:20
+array(
+)

test/code/parser/errorHandling/recovery.test

@@ -0,0 +1,186 @@
+Error recovery
+-----
+<?php
+
+foo()
+bar()
+baz()
+-----
+Syntax error, unexpected T_STRING from 4:1 to 4:3
+Syntax error, unexpected T_STRING from 5:1 to 5:3
+Syntax error, unexpected EOF from 5:6 to 5:6
+array(
+)
+-----
+<?php
+
+foo()
+bar();
+baz();
+-----
+Syntax error, unexpected T_STRING from 4:1 to 4:3
+array(
+    0: Expr_FuncCall(
+        name: Name(
+            parts: array(
+                0: bar
+            )
+        )
+        args: array(
+        )
+    )
+    1: Expr_FuncCall(
+        name: Name(
+            parts: array(
+                0: baz
+            )
+        )
+        args: array(
+        )
+    )
+)
+-----
+<?php
+
+foo();
+bar()
+baz();
+-----
+Syntax error, unexpected T_STRING from 5:1 to 5:3
+array(
+    0: Expr_FuncCall(
+        name: Name(
+            parts: array(
+                0: foo
+            )
+        )
+        args: array(
+        )
+    )
+    1: Expr_FuncCall(
+        name: Name(
+            parts: array(
+                0: baz
+            )
+        )
+        args: array(
+        )
+    )
+)
+-----
+<?php
+abc;
+1 + ;
+-----
+Syntax error, unexpected ';' from 3:5 to 3:5
+array(
+    0: Expr_ConstFetch(
+        name: Name(
+            parts: array(
+                0: abc
+            )
+        )
+    )
+)
+-----
+<?php
+function test() {
+    1 +
+}
+-----
+Syntax error, unexpected '}' from 4:1 to 4:1
+array(
+    0: Stmt_Function(
+        byRef: false
+        name: test
+        params: array(
+        )
+        returnType: null
+        stmts: array(
+        )
+    )
+)
+-----
+<?php
+
+$i = 0;
+while
+
+$j = 1;
+$k = 2;
+-----
+Syntax error, unexpected T_VARIABLE, expecting '(' from 6:1 to 6:2
+array(
+    0: Expr_Assign(
+        var: Expr_Variable(
+            name: i
+        )
+        expr: Scalar_LNumber(
+            value: 0
+        )
+    )
+    1: Expr_Assign(
+        var: Expr_Variable(
+            name: j
+        )
+        expr: Scalar_LNumber(
+            value: 1
+        )
+    )
+    2: Expr_Assign(
+        var: Expr_Variable(
+            name: k
+        )
+        expr: Scalar_LNumber(
+            value: 2
+        )
+    )
+)
+-----
+<?php
+
+$i = 0;
+while () {
+    $j = 1;
+}
+$k = 2;
+// The output here drops the loop - would require Error node to handle this
+-----
+Syntax error, unexpected ')' from 4:8 to 4:8
+array(
+    0: Expr_Assign(
+        var: Expr_Variable(
+            name: i
+        )
+        expr: Scalar_LNumber(
+            value: 0
+        )
+    )
+    1: Expr_Assign(
+        var: Expr_Variable(
+            name: j
+        )
+        expr: Scalar_LNumber(
+            value: 1
+        )
+    )
+    2: Expr_Assign(
+        var: Expr_Variable(
+            name: k
+        )
+        expr: Scalar_LNumber(
+            value: 2
+        )
+    )
+)
+-----
+<?php
+// Can't recover this yet, as the '}' for the inner_statement_list
+// is always required.
+
+$i = 0;
+while (true) {
+    $i = 1;
+    $i = 2;
+-----
+Syntax error, unexpected EOF from 8:12 to 8:12

test/code/parser/expr/newWithoutClass.test

@@ -0,0 +1,8 @@
+New without a class
+-----
+<?php
+new;
+-----
+Syntax error, unexpected ';' from 2:4 to 2:4
+array(
+)

test/code/parser/expr/newWithoutClass.test-fail

@@ -1,6 +0,0 @@
-New without a class
------
-<?php
-new;
------
-Syntax error, unexpected ';' on line 2

test/code/parser/scalar/float.test

@@ -18,6 +18,7 @@ Different float syntaxes
 18446744073709551615;
 0xFFFFFFFFFFFFFFFF;
 01777777777777777777777;
+0177777777777777777777787;
 0b1111111111111111111111111111111111111111111111111111111111111111;
 -----
 array(
@@ -63,4 +64,7 @@ array(
     13: Scalar_DNumber(
         value: @@{ 0xFFFFFFFFFFFFFFFF }@@
     )
+    14: Scalar_DNumber(
+        value: @@{ 0xFFFFFFFFFFFFFFFF }@@
+    )
 )

test/code/parser/scalar/int.test

@@ -10,6 +10,7 @@ Different integer syntaxes
 0xfff;
 0XfFf;
 0777;
+0787;
 0b111000111000;
 -----
 array(
@@ -38,6 +39,9 @@ array(
         value: 511
     )
     8: Scalar_LNumber(
+        value: 7
+    )
+    9: Scalar_LNumber(
         value: 3640
     )
 )

test/code/parser/stmt/class/anonymous.test

@@ -0,0 +1,194 @@
+Anonymous classes
+-----
+<?php
+
+new class {
+    public function test() {}
+};
+new class extends A implements B, C {};
+new class() {
+    public $foo;
+};
+new class($a, $b) extends A {
+    use T;
+};
+
+class A {
+    public function test() {
+        return new class($this) extends A {
+            const A = 'B';
+        };
+    }
+}
+-----
+array(
+    0: Expr_New(
+        class: Stmt_Class(
+            type: 0
+            name: null
+            extends: null
+            implements: array(
+            )
+            stmts: array(
+                0: Stmt_ClassMethod(
+                    type: 1
+                    byRef: false
+                    name: test
+                    params: array(
+                    )
+                    returnType: null
+                    stmts: array(
+                    )
+                )
+            )
+        )
+        args: array(
+        )
+    )
+    1: Expr_New(
+        class: Stmt_Class(
+            type: 0
+            name: null
+            extends: Name(
+                parts: array(
+                    0: A
+                )
+            )
+            implements: array(
+                0: Name(
+                    parts: array(
+                        0: B
+                    )
+                )
+                1: Name(
+                    parts: array(
+                        0: C
+                    )
+                )
+            )
+            stmts: array(
+            )
+        )
+        args: array(
+        )
+    )
+    2: Expr_New(
+        class: Stmt_Class(
+            type: 0
+            name: null
+            extends: null
+            implements: array(
+            )
+            stmts: array(
+                0: Stmt_Property(
+                    type: 1
+                    props: array(
+                        0: Stmt_PropertyProperty(
+                            name: foo
+                            default: null
+                        )
+                    )
+                )
+            )
+        )
+        args: array(
+        )
+    )
+    3: Expr_New(
+        class: Stmt_Class(
+            type: 0
+            name: null
+            extends: Name(
+                parts: array(
+                    0: A
+                )
+            )
+            implements: array(
+            )
+            stmts: array(
+                0: Stmt_TraitUse(
+                    traits: array(
+                        0: Name(
+                            parts: array(
+                                0: T
+                            )
+                        )
+                    )
+                    adaptations: array(
+                    )
+                )
+            )
+        )
+        args: array(
+            0: Arg(
+                value: Expr_Variable(
+                    name: a
+                )
+                byRef: false
+                unpack: false
+            )
+            1: Arg(
+                value: Expr_Variable(
+                    name: b
+                )
+                byRef: false
+                unpack: false
+            )
+        )
+    )
+    4: Stmt_Class(
+        type: 0
+        name: A
+        extends: null
+        implements: array(
+        )
+        stmts: array(
+            0: Stmt_ClassMethod(
+                type: 1
+                byRef: false
+                name: test
+                params: array(
+                )
+                returnType: null
+                stmts: array(
+                    0: Stmt_Return(
+                        expr: Expr_New(
+                            class: Stmt_Class(
+                                type: 0
+                                name: null
+                                extends: Name(
+                                    parts: array(
+                                        0: A
+                                    )
+                                )
+                                implements: array(
+                                )
+                                stmts: array(
+                                    0: Stmt_ClassConst(
+                                        consts: array(
+                                            0: Const(
+                                                name: A
+                                                value: Scalar_String(
+                                                    value: B
+                                                )
+                                            )
+                                        )
+                                    )
+                                )
+                            )
+                            args: array(
+                                0: Arg(
+                                    value: Expr_Variable(
+                                        name: this
+                                    )
+                                    byRef: false
+                                    unpack: false
+                                )
+                            )
+                        )
+                    )
+                )
+            )
+        )
+    )
+)

test/code/parser/stmt/class/modifier.test

@@ -25,8 +25,20 @@ Multiple final modifiers are not allowed on line 1
 Cannot use the final modifier on an abstract class member on line 1
 -----
 <?php abstract final class A { }
+// Type in the partial parse could conceivably be any of 0, 16 or 32
 -----
-Syntax error, unexpected T_FINAL, expecting T_CLASS on line 1
+Syntax error, unexpected T_FINAL, expecting T_CLASS from 1:16 to 1:20
+array(
+    0: Stmt_Class(
+        type: 32
+        name: A
+        extends: null
+        implements: array(
+        )
+        stmts: array(
+        )
+    )
+)
 -----
 <?php class A { abstract $a; }
 -----

test/code/parser/stmt/class/name.test

@@ -0,0 +1,71 @@
+Invalid class name
+-----
+<?php class self {}
+-----
+Cannot use 'self' as class name as it is reserved on line 1
+-----
+<?php class PARENT {}
+-----
+Cannot use 'PARENT' as class name as it is reserved on line 1
+-----
+<?php class static {}
+-----
+Syntax error, unexpected T_STATIC, expecting T_STRING from 1:13 to 1:18
+array(
+)
+-----
+<?php class A extends self {}
+-----
+Cannot use 'self' as class name as it is reserved from 1:23 to 1:26
+-----
+<?php class A extends PARENT {}
+-----
+Cannot use 'PARENT' as class name as it is reserved from 1:23 to 1:28
+-----
+<?php class A extends static {}
+-----
+Syntax error, unexpected T_STATIC, expecting T_STRING or T_NAMESPACE or T_NS_SEPARATOR from 1:23 to 1:28
+array(
+)
+-----
+<?php class A implements self {}
+-----
+Cannot use 'self' as interface name as it is reserved from 1:26 to 1:29
+-----
+<?php class A implements PARENT {}
+-----
+Cannot use 'PARENT' as interface name as it is reserved from 1:26 to 1:31
+-----
+<?php class A implements static {}
+-----
+Syntax error, unexpected T_STATIC, expecting T_STRING or T_NAMESPACE or T_NS_SEPARATOR from 1:26 to 1:31
+array(
+)
+-----
+<?php interface self {}
+-----
+Cannot use 'self' as class name as it is reserved on line 1
+-----
+<?php interface PARENT {}
+-----
+Cannot use 'PARENT' as class name as it is reserved on line 1
+-----
+<?php interface static {}
+-----
+Syntax error, unexpected T_STATIC, expecting T_STRING from 1:17 to 1:22
+array(
+)
+-----
+<?php interface A extends self {}
+-----
+Cannot use 'self' as interface name as it is reserved from 1:27 to 1:30
+-----
+<?php interface A extends PARENT {}
+-----
+Cannot use 'PARENT' as interface name as it is reserved from 1:27 to 1:32
+-----
+<?php interface A extends static {}
+-----
+Syntax error, unexpected T_STATIC, expecting T_STRING or T_NAMESPACE or T_NS_SEPARATOR from 1:27 to 1:32
+array(
+)

test/code/parser/stmt/class/name.test-fail

@@ -1,61 +0,0 @@
-Invalid class name
------
-<?php class self {}
------
-Cannot use 'self' as class name as it is reserved on line 1
------
-<?php class parent {}
------
-Cannot use 'parent' as class name as it is reserved on line 1
------
-<?php class static {}
------
-Syntax error, unexpected T_STATIC, expecting T_STRING on line 1
------
-<?php class A extends self {}
------
-Cannot use 'self' as class name as it is reserved on line 1
------
-<?php class A extends parent {}
------
-Cannot use 'parent' as class name as it is reserved on line 1
------
-<?php class A extends static {}
------
-Syntax error, unexpected T_STATIC, expecting T_STRING or T_NAMESPACE or T_NS_SEPARATOR on line 1
------
-<?php class A implements self {}
------
-Cannot use 'self' as interface name as it is reserved on line 1
------
-<?php class A implements parent {}
------
-Cannot use 'parent' as interface name as it is reserved on line 1
------
-<?php class A implements static {}
------
-Syntax error, unexpected T_STATIC, expecting T_STRING or T_NAMESPACE or T_NS_SEPARATOR on line 1
------
-<?php interface self {}
------
-Cannot use 'self' as class name as it is reserved on line 1
------
-<?php interface parent {}
------
-Cannot use 'parent' as class name as it is reserved on line 1
------
-<?php interface static {}
------
-Syntax error, unexpected T_STATIC, expecting T_STRING on line 1
------
-<?php interface A extends self {}
------
-Cannot use 'self' as interface name as it is reserved on line 1
------
-<?php interface A extends parent {}
------
-Cannot use 'parent' as interface name as it is reserved on line 1
------
-<?php interface A extends static {}
------
-Syntax error, unexpected T_STATIC, expecting T_STRING or T_NAMESPACE or T_NS_SEPARATOR on line 1

test/code/parser/stmt/function/generator.test

@@ -1,4 +1,4 @@
-Generators (yield expression
+Generators (yield expression)
 -----
 <?php
 
@@ -25,6 +25,10 @@ function gen() {
     func(yield $foo);
     $foo->func(yield $foo);
     new Foo(yield $foo);
+
+    yield from $foo;
+    yield from $foo and yield from $bar;
+    yield from $foo + $bar;
 }
 -----
 array(
@@ -226,6 +230,33 @@ array(
                     )
                 )
             )
+            15: Expr_YieldFrom(
+                expr: Expr_Variable(
+                    name: foo
+                )
+            )
+            16: Expr_BinaryOp_LogicalAnd(
+                left: Expr_YieldFrom(
+                    expr: Expr_Variable(
+                        name: foo
+                    )
+                )
+                right: Expr_YieldFrom(
+                    expr: Expr_Variable(
+                        name: bar
+                    )
+                )
+            )
+            17: Expr_YieldFrom(
+                expr: Expr_BinaryOp_Plus(
+                    left: Expr_Variable(
+                        name: foo
+                    )
+                    right: Expr_Variable(
+                        name: bar
+                    )
+                )
+            )
         )
     )
 )

test/code/parser/stmt/function/variadicDefaultValue.test

@@ -3,4 +3,4 @@ Invalid variadic function
 <?php
 function foo(...$foo = []) {}
 -----
-Variadic parameter cannot have a default value on line 2
+Variadic parameter cannot have a default value from 2:24 to 2:25

test/code/parser/stmt/haltCompilerOffset.test

@@ -0,0 +1,34 @@
+Use of __HALT_COMPILER_OFFSET__ constant
+-----
+<?php
+
+var_dump(__HALT_COMPILER_OFFSET__);
+__halt_compiler();
+Foo
+-----
+array(
+    0: Expr_FuncCall(
+        name: Name(
+            parts: array(
+                0: var_dump
+            )
+        )
+        args: array(
+            0: Arg(
+                value: Expr_ConstFetch(
+                    name: Name(
+                        parts: array(
+                            0: __HALT_COMPILER_OFFSET__
+                        )
+                    )
+                )
+                byRef: false
+                unpack: false
+            )
+        )
+    )
+    1: Stmt_HaltCompiler(
+        remaining:
+    Foo
+    )
+)

test/code/parser/stmt/haltCompilerOutermostScope.test

@@ -5,4 +5,4 @@ if (true) {
     __halt_compiler();
 }
 -----
-__HALT_COMPILER() can only be used from the outermost scope on line 3
+__HALT_COMPILER() can only be used from the outermost scope from 3:5 to 3:19

test/code/parser/stmt/namespace/invalidName.test

@@ -0,0 +1,29 @@
+Invalid namespace names
+-----
+<?php namespace self;
+-----
+Cannot use 'self' as namespace name from 1:17 to 1:20
+-----
+<?php namespace PARENT;
+-----
+Cannot use 'PARENT' as namespace name from 1:17 to 1:22
+-----
+<?php namespace static;
+-----
+Syntax error, unexpected T_STATIC, expecting T_STRING or T_NS_SEPARATOR or '{' from 1:17 to 1:22
+array(
+)
+-----
+<?php use A as self;
+-----
+Cannot use A as self because 'self' is a special class name on line 1
+-----
+<?php use B as PARENT;
+-----
+Cannot use B as PARENT because 'PARENT' is a special class name on line 1
+-----
+<?php use C as static;
+-----
+Syntax error, unexpected T_STATIC, expecting T_STRING from 1:16 to 1:21
+array(
+)

test/code/parser/stmt/namespace/name.test-fail

@@ -1,25 +0,0 @@
-Invalid namespace names
------
-<?php namespace self;
------
-Cannot use 'self' as namespace name on line 1
------
-<?php namespace parent;
------
-Cannot use 'parent' as namespace name on line 1
------
-<?php namespace static;
------
-Syntax error, unexpected T_STATIC, expecting T_STRING or T_NS_SEPARATOR or '{' on line 1
------
-<?php use A as self;
------
-Cannot use A as self because 'self' is a special class name on line 1
------
-<?php use B as parent;
------
-Cannot use B as parent because 'parent' is a special class name on line 1
------
-<?php use C as static;
------
-Syntax error, unexpected T_STATIC, expecting T_STRING on line 1

test/code/parser/stmt/namespace/nested.test

@@ -7,4 +7,4 @@ namespace A {
     }
 }
 -----
-Namespace declarations cannot be nested on line 3
+Namespace declarations cannot be nested from 3:5 to 5:5

test/code/prettyPrinter/anonymousClass.test

@@ -0,0 +1,27 @@
+Anonymous classes
+-----
+<?php
+
+new class {};
+new class extends A implements B, C {};
+new class($a) extends A {
+    private $a;
+    public function __construct($a) {
+        $this->a = $a;
+    }
+};
+-----
+new class
+{
+};
+new class extends A implements B, C
+{
+};
+new class($a) extends A
+{
+    private $a;
+    public function __construct($a)
+    {
+        $this->a = $a;
+    }
+};

test/code/prettyPrinter/literals.test

@@ -37,6 +37,8 @@ FALSE;
 1e1000;
 1E-100;
 1000000000000000000000000000000000000000000000000000000000000000000000000000000000000;
+378282246310005.0;
+10000000000000002.0;
 
 // strings (normalized to single quoted)
 'a';
@@ -112,6 +114,8 @@ FALSE;
 INF;
 1.0E-100;
 1.0E+84;
+378282246310005.0;
+10000000000000002.0;
 // strings (normalized to single quoted)
 'a';
 'a

test/code/prettyPrinter/parentheses.test

@@ -33,6 +33,11 @@ $a ** $b ** $c;
 ($a ** $b) ** $c;
 -1 ** 2;
 
+yield from $a and yield from $b;
+yield from ($a and yield from $b);
+
+print ($a and print $b);
+
 // The following will currently add unnecessary parentheses, because the pretty printer is not aware that assignment
 // and incdec only work on variables.
 !$a = $b;
@@ -62,6 +67,9 @@ $a + $b++;
 $a ** $b ** $c;
 ($a ** $b) ** $c;
 -1 ** 2;
+yield from $a and yield from $b;
+yield from ($a and yield from $b);
+print ($a and print $b);
 // The following will currently add unnecessary parentheses, because the pretty printer is not aware that assignment
 // and incdec only work on variables.
 !($a = $b);