Release PHP-Parser 3.0.6

I encounter an issue when no tokens exist. An error `Undefined offset -1` is triggered.

.travis.yml

@@ -1,5 +1,5 @@
 language: php
-
+dist: trusty
 sudo: false
 
 cache:

CHANGELOG.md

@@ -1,8 +1,38 @@
-Version 3.0.5-dev
+Version 3.1.0-dev
 -----------------
 
 Nothing yet.
 
+Version 3.0.6 (2017-06-28)
+--------------------------
+
+### Fixed
+
+* Fixed the spelling of `Class_::VISIBILITY_MODIFIER_MASK`. The previous spelling of
+  `Class_::VISIBILITY_MODIFER_MASK` is preserved for backwards compatibility.
+* The pretty printing will now preserve comments inside array literals and function calls by
+  printing the array items / function arguments on separate lines. Array literals and functions that
+  do not contain comments are not affected.
+
+### Added
+
+* Added `Builder\Param::makeVariadic()`.
+
+### Deprecated
+
+* The `Node::setLine()` method has been deprecated.
+
+Version 3.0.5 (2017-03-05)
+--------------------------
+
+### Fixed
+
+* Name resolution of `NullableType`s is now performed earlier, so that a fully resolved signature is
+  available when a function is entered. (#360)
+* `Error` nodes are now considered empty, while previously they extended until the token where the
+  error occurred. This made some nodes larger than expected. (#359)
+* Fixed notices being thrown during error recovery in some situations. (#362)
+
 Version 3.0.4 (2017-02-10)
 --------------------------
 

grammar/php7.y

@@ -763,7 +763,7 @@ constant:
     /* We interpret and isolated FOO:: as an unfinished class constant fetch. It could also be
        an unfinished static property fetch or unfinished scoped call. */
     | class_name_or_var T_PAAMAYIM_NEKUDOTAYIM error
-          { $$ = Expr\ClassConstFetch[$1, Expr\Error[]]; $this->errorState = 2; }
+          { $$ = Expr\ClassConstFetch[$1, new Expr\Error(stackAttributes(#3))]; $this->errorState = 2; }
 ;
 
 array_short_syntax:

lib/PhpParser/Builder/Param.php

@@ -16,6 +16,8 @@ class Param extends PhpParser\BuilderAbstract
 
     protected $byRef = false;
 
+    protected $variadic = false;
+
     /**
      * Creates a parameter builder.
      *
@@ -65,6 +67,17 @@ class Param extends PhpParser\BuilderAbstract
         return $this;
     }
 
+    /**
+     * Make the parameter variadic
+     *
+     * @return $this The builder instance (for fluid interface)
+     */
+    public function makeVariadic() {
+        $this->variadic = true;
+
+        return $this;
+    }
+
     /**
      * Returns the built parameter node.
      *
@@ -72,7 +85,7 @@ class Param extends PhpParser\BuilderAbstract
      */
     public function getNode() {
         return new Node\Param(
-            $this->name, $this->default, $this->type, $this->byRef
+            $this->name, $this->default, $this->type, $this->byRef, $this->variadic
         );
     }
 }

lib/PhpParser/Lexer.php

@@ -185,15 +185,17 @@ class Lexer
             return;
         }
 
-        // Check for unterminated comment
+        if (count($this->tokens) > 0) {
-        $lastToken = $this->tokens[count($this->tokens) - 1];
+            // Check for unterminated comment
-        if ($this->isUnterminatedComment($lastToken)) {
+            $lastToken = $this->tokens[count($this->tokens) - 1];
-            $errorHandler->handleError(new Error('Unterminated comment', [
+            if ($this->isUnterminatedComment($lastToken)) {
-                'startLine' => $line - substr_count($lastToken[1], "\n"),
+                $errorHandler->handleError(new Error('Unterminated comment', [
-                'endLine' => $line,
+                    'startLine' => $line - substr_count($lastToken[1], "\n"),
-                'startFilePos' => $filePos - \strlen($lastToken[1]),
+                    'endLine' => $line,
-                'endFilePos' => $filePos,
+                    'startFilePos' => $filePos - \strlen($lastToken[1]),
-            ]));
+                    'endFilePos' => $filePos,
+                ]));
+            }
         }
     }
 

lib/PhpParser/Node.php

@@ -29,6 +29,8 @@ interface Node
      * Sets line the node started in.
      *
      * @param int $line Line
+     *
+     * @deprecated
      */
     public function setLine($line);
 

lib/PhpParser/Node/Stmt/ClassConst.php

@@ -30,7 +30,7 @@ class ClassConst extends Node\Stmt
 
     public function isPublic() {
         return ($this->flags & Class_::MODIFIER_PUBLIC) !== 0
-            || ($this->flags & Class_::VISIBILITY_MODIFER_MASK) === 0;
+            || ($this->flags & Class_::VISIBILITY_MODIFIER_MASK) === 0;
     }
 
     public function isProtected() {

lib/PhpParser/Node/Stmt/ClassMethod.php

@@ -69,7 +69,7 @@ class ClassMethod extends Node\Stmt implements FunctionLike
 
     public function isPublic() {
         return ($this->flags & Class_::MODIFIER_PUBLIC) !== 0
-            || ($this->flags & Class_::VISIBILITY_MODIFER_MASK) === 0;
+            || ($this->flags & Class_::VISIBILITY_MODIFIER_MASK) === 0;
     }
 
     public function isProtected() {

lib/PhpParser/Node/Stmt/Class_.php

@@ -14,7 +14,9 @@ class Class_ extends ClassLike
     const MODIFIER_ABSTRACT  = 16;
     const MODIFIER_FINAL     = 32;
 
-    const VISIBILITY_MODIFER_MASK = 7; // 1 | 2 | 4
+    const VISIBILITY_MODIFIER_MASK = 7; // 1 | 2 | 4
+    /** @deprecated */
+    const VISIBILITY_MODIFER_MASK = self::VISIBILITY_MODIFIER_MASK;
 
     /** @var int Type */
     public $flags;
@@ -74,7 +76,7 @@ class Class_ extends ClassLike
      * @internal
      */
     public static function verifyModifier($a, $b) {
-        if ($a & self::VISIBILITY_MODIFER_MASK && $b & self::VISIBILITY_MODIFER_MASK) {
+        if ($a & self::VISIBILITY_MODIFIER_MASK && $b & self::VISIBILITY_MODIFIER_MASK) {
             throw new Error('Multiple access type modifiers are not allowed');
         }
 

lib/PhpParser/Node/Stmt/Property.php

@@ -34,7 +34,7 @@ class Property extends Node\Stmt
 
     public function isPublic() {
         return ($this->flags & Class_::MODIFIER_PUBLIC) !== 0
-            || ($this->flags & Class_::VISIBILITY_MODIFER_MASK) === 0;
+            || ($this->flags & Class_::VISIBILITY_MODIFIER_MASK) === 0;
     }
 
     public function isProtected() {

lib/PhpParser/NodeAbstract.php

@@ -37,6 +37,8 @@ abstract class NodeAbstract implements Node, \JsonSerializable
      * Sets line the node started in.
      *
      * @param int $line Line
+     *
+     * @deprecated
      */
     public function setLine($line) {
         $this->setAttribute('startLine', (int) $line);

lib/PhpParser/NodeVisitor/NameResolver.php

@@ -120,10 +120,6 @@ class NameResolver extends NodeVisitorAbstract
                     }
                 }
             }
-        } elseif ($node instanceof Node\NullableType) {
-            if ($node->type instanceof Name) {
-                $node->type = $this->resolveClassName($node->type);
-            }
         }
     }
 
@@ -172,13 +168,20 @@ class NameResolver extends NodeVisitorAbstract
     /** @param Stmt\Function_|Stmt\ClassMethod|Expr\Closure $node */
     private function resolveSignature($node) {
         foreach ($node->params as $param) {
-            if ($param->type instanceof Name) {
+            $param->type = $this->resolveType($param->type);
-                $param->type = $this->resolveClassName($param->type);
-            }
         }
-        if ($node->returnType instanceof Name) {
+        $node->returnType = $this->resolveType($node->returnType);
-            $node->returnType = $this->resolveClassName($node->returnType);
+    }
+
+    private function resolveType($node) {
+        if ($node instanceof Node\NullableType) {
+            $node->type = $this->resolveType($node->type);
+            return $node;
         }
+        if ($node instanceof Name) {
+            return $this->resolveClassName($node);
+        }
+        return $node;
     }
 
     protected function resolveClassName(Name $name) {

lib/PhpParser/Parser/Php7.php

@@ -2483,7 +2483,7 @@ class Php7 extends \PhpParser\ParserAbstract
     }
 
     protected function reduceRule416() {
-         $this->semValue = new Expr\ClassConstFetch($this->semStack[$this->stackPos-(3-1)], new Expr\Error($this->startAttributeStack[$this->stackPos-(3-1)] + $this->endAttributes), $this->startAttributeStack[$this->stackPos-(3-1)] + $this->endAttributes); $this->errorState = 2;
+         $this->semValue = new Expr\ClassConstFetch($this->semStack[$this->stackPos-(3-1)], new Expr\Error($this->startAttributeStack[$this->stackPos-(3-3)] + $this->endAttributeStack[$this->stackPos-(3-3)]), $this->startAttributeStack[$this->stackPos-(3-1)] + $this->endAttributes); $this->errorState = 2;
     }
 
     protected function reduceRule417() {

lib/PhpParser/ParserAbstract.php

@@ -315,7 +315,11 @@ abstract class ParserAbstract implements Parser
                             //$this->traceShift($this->errorSymbol);
                             ++$this->stackPos;
                             $stateStack[$this->stackPos] = $state = $action;
-                            $this->endAttributes = $this->endAttributeStack[$this->stackPos];
+
+                            // We treat the error symbol as being empty, so we reset the end attributes
+                            // to the end attributes of the last non-error symbol
+                            $this->endAttributeStack[$this->stackPos] = $this->endAttributeStack[$this->stackPos - 1];
+                            $this->endAttributes = $this->endAttributeStack[$this->stackPos - 1];
                             break;
 
                         case 3:

lib/PhpParser/PrettyPrinter/Standard.php

@@ -438,12 +438,12 @@ class Standard extends PrettyPrinterAbstract
 
     protected function pExpr_FuncCall(Expr\FuncCall $node) {
         return $this->pCallLhs($node->name)
-             . '(' . $this->pCommaSeparated($node->args) . ')';
+             . '(' . $this->pMaybeMultiline($node->args) . ')';
     }
 
     protected function pExpr_MethodCall(Expr\MethodCall $node) {
         return $this->pDereferenceLhs($node->var) . '->' . $this->pObjectProperty($node->name)
-             . '(' . $this->pCommaSeparated($node->args) . ')';
+             . '(' . $this->pMaybeMultiline($node->args) . ')';
     }
 
     protected function pExpr_StaticCall(Expr\StaticCall $node) {
@@ -453,7 +453,7 @@ class Standard extends PrettyPrinterAbstract
                    ? $this->p($node->name)
                    : '{' . $this->p($node->name) . '}')
                 : $node->name)
-             . '(' . $this->pCommaSeparated($node->args) . ')';
+             . '(' . $this->pMaybeMultiline($node->args) . ')';
     }
 
     protected function pExpr_Empty(Expr\Empty_ $node) {
@@ -501,9 +501,9 @@ class Standard extends PrettyPrinterAbstract
         $syntax = $node->getAttribute('kind',
             $this->options['shortArraySyntax'] ? Expr\Array_::KIND_SHORT : Expr\Array_::KIND_LONG);
         if ($syntax === Expr\Array_::KIND_SHORT) {
-            return '[' . $this->pCommaSeparated($node->items) . ']';
+            return '[' . $this->pMaybeMultiline($node->items, true) . ']';
         } else {
-            return 'array(' . $this->pCommaSeparated($node->items) . ')';
+            return 'array(' . $this->pMaybeMultiline($node->items, true) . ')';
         }
     }
 
@@ -553,10 +553,10 @@ class Standard extends PrettyPrinterAbstract
 
     protected function pExpr_New(Expr\New_ $node) {
         if ($node->class instanceof Stmt\Class_) {
-            $args = $node->args ? '(' . $this->pCommaSeparated($node->args) . ')' : '';
+            $args = $node->args ? '(' . $this->pMaybeMultiline($node->args) . ')' : '';
             return 'new ' . $this->pClassCommon($node->class, $args);
         }
-        return 'new ' . $this->p($node->class) . '(' . $this->pCommaSeparated($node->args) . ')';
+        return 'new ' . $this->p($node->class) . '(' . $this->pMaybeMultiline($node->args) . ')';
     }
 
     protected function pExpr_Clone(Expr\Clone_ $node) {
@@ -944,4 +944,21 @@ class Standard extends PrettyPrinterAbstract
             return '(' . $this->p($node) . ')';
         }
     }
+
+    private function hasNodeWithComments(array $nodes) {
+        foreach ($nodes as $node) {
+            if ($node && $node->getAttribute('comments')) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private function pMaybeMultiline(array $nodes, $trailingComma = false) {
+        if (!$this->hasNodeWithComments($nodes)) {
+            return $this->pCommaSeparated($nodes);
+        } else {
+            return $this->pCommaSeparatedMultiline($nodes, $trailingComma) . "\n";
+        }
+    }
 }

lib/PhpParser/PrettyPrinterAbstract.php

@@ -286,6 +286,38 @@ abstract class PrettyPrinterAbstract
         return $this->pImplode($nodes, ', ');
     }
 
+    /**
+     * Pretty prints a comma-separated list of nodes in multiline style, including comments.
+     *
+     * The result includes a leading newline and one level of indentation (same as pStmts).
+     *
+     * @param Node[] $nodes         Array of Nodes to be printed
+     * @param bool   $trailingComma Whether to use a trailing comma
+     *
+     * @return string Comma separated pretty printed nodes in multiline style
+     */
+    protected function pCommaSeparatedMultiline(array $nodes, $trailingComma) {
+        $result = '';
+        $lastIdx = count($nodes) - 1;
+        foreach ($nodes as $idx => $node) {
+            if ($node !== null) {
+                $comments = $node->getAttribute('comments', array());
+                if ($comments) {
+                    $result .= "\n" . $this->pComments($comments);
+                }
+
+                $result .= "\n" . $this->p($node);
+            } else {
+                $result .= "\n";
+            }
+            if ($trailingComma || $idx !== $lastIdx) {
+                $result .= ',';
+            }
+        }
+
+        return preg_replace('~\n(?!$|' . $this->noIndentToken . ')~', "\n    ", $result);
+    }
+
     /**
      * Signals the pretty printer that a string shall not be indented.
      *

test/PhpParser/Builder/ParamTest.php

@@ -155,4 +155,16 @@ class ParamTest extends \PHPUnit_Framework_TestCase
             $node
         );
     }
+
+    public function testVariadic() {
+        $node = $this->createParamBuilder('test')
+            ->makeVariadic()
+            ->getNode()
+        ;
+
+        $this->assertEquals(
+            new Node\Param('test', null, null, false, true),
+            $node
+        );
+    }
 }

test/PhpParser/LexerTest.php

@@ -201,7 +201,13 @@ class LexerTest extends \PHPUnit_Framework_TestCase
                         array(), array()
                     )
                 )
-            )
+            ),
+            // tests no tokens
+            array(
+                '',
+                array(),
+                array()
+            ),
         );
     }
 

test/PhpParser/NodeTraverserTest.php

@@ -178,7 +178,7 @@ class NodeTraverserTest extends \PHPUnit_Framework_TestCase
         $visitor = $this->getMockBuilder('PhpParser\NodeVisitor')->getMock();
         $visitor->expects($this->at(1))->method('enterNode')->with($mulNode)
             ->will($this->returnValue(NodeTraverser::STOP_TRAVERSAL));
-        $visitor->expects($this->at(2))->method('afterTraversal');
+        $visitor->expects($this->at(2))->method('afterTraverse');
         $traverser = new NodeTraverser;
         $traverser->addVisitor($visitor);
         $this->assertEquals($stmts, $traverser->traverse($stmts));
@@ -187,7 +187,7 @@ class NodeTraverserTest extends \PHPUnit_Framework_TestCase
         $visitor = $this->getMockBuilder('PhpParser\NodeVisitor')->getMock();
         $visitor->expects($this->at(2))->method('enterNode')->with($varNode1)
             ->will($this->returnValue(NodeTraverser::STOP_TRAVERSAL));
-        $visitor->expects($this->at(3))->method('afterTraversal');
+        $visitor->expects($this->at(3))->method('afterTraverse');
         $traverser = new NodeTraverser;
         $traverser->addVisitor($visitor);
         $this->assertEquals($stmts, $traverser->traverse($stmts));
@@ -196,7 +196,7 @@ class NodeTraverserTest extends \PHPUnit_Framework_TestCase
         $visitor = $this->getMockBuilder('PhpParser\NodeVisitor')->getMock();
         $visitor->expects($this->at(3))->method('leaveNode')->with($varNode1)
             ->will($this->returnValue(NodeTraverser::STOP_TRAVERSAL));
-        $visitor->expects($this->at(4))->method('afterTraversal');
+        $visitor->expects($this->at(4))->method('afterTraverse');
         $traverser = new NodeTraverser;
         $traverser->addVisitor($visitor);
         $this->assertEquals($stmts, $traverser->traverse($stmts));
@@ -205,7 +205,7 @@ class NodeTraverserTest extends \PHPUnit_Framework_TestCase
         $visitor = $this->getMockBuilder('PhpParser\NodeVisitor')->getMock();
         $visitor->expects($this->at(6))->method('leaveNode')->with($mulNode)
             ->will($this->returnValue(NodeTraverser::STOP_TRAVERSAL));
-        $visitor->expects($this->at(7))->method('afterTraversal');
+        $visitor->expects($this->at(7))->method('afterTraverse');
         $traverser = new NodeTraverser;
         $traverser->addVisitor($visitor);
         $this->assertEquals($stmts, $traverser->traverse($stmts));
@@ -216,7 +216,7 @@ class NodeTraverserTest extends \PHPUnit_Framework_TestCase
             ->will($this->returnValue(NodeTraverser::REMOVE_NODE));
         $visitor->expects($this->at(7))->method('enterNode')->with($printNode)
             ->will($this->returnValue(NodeTraverser::STOP_TRAVERSAL));
-        $visitor->expects($this->at(8))->method('afterTraversal');
+        $visitor->expects($this->at(8))->method('afterTraverse');
         $traverser = new NodeTraverser;
         $traverser->addVisitor($visitor);
         $this->assertEquals([$printNode], $traverser->traverse($stmts));

test/code/parser/errorHandling/recovery.test

@@ -248,11 +248,11 @@ $foo->
 !!positions
 Syntax error, unexpected ';', expecting T_STRING or T_VARIABLE or '{' or '$' from 3:1 to 3:1
 array(
-    0: Expr_PropertyFetch[2:1 - 3:1](
+    0: Expr_PropertyFetch[2:1 - 2:6](
         var: Expr_Variable[2:1 - 2:4](
             name: foo
         )
-        name: Expr_Error[3:1 - 3:1](
+        name: Expr_Error[3:1 - 2:6](
         )
     )
 )
@@ -272,11 +272,11 @@ array(
         )
         returnType: null
         stmts: array(
-            0: Expr_PropertyFetch[3:5 - 4:1](
+            0: Expr_PropertyFetch[3:5 - 3:10](
                 var: Expr_Variable[3:5 - 3:8](
                     name: bar
                 )
-                name: Expr_Error[4:1 - 4:1](
+                name: Expr_Error[4:1 - 3:10](
                 )
             )
         )
@@ -305,8 +305,8 @@ new
 !!php7,positions
 Syntax error, unexpected EOF from 2:4 to 2:4
 array(
-    0: Expr_New[2:1 - 2:4](
+    0: Expr_New[2:1 - 2:3](
-        class: Expr_Error[2:4 - 2:4](
+        class: Expr_Error[2:4 - 2:3](
         )
         args: array(
         )
@@ -834,4 +834,35 @@ array(
         stmts: array(
         )
     )
+)
+-----
+<?php
+
+foo(Bar::);
+-----
+!!php7,positions
+Syntax error, unexpected ')' from 3:10 to 3:10
+array(
+    0: Expr_FuncCall[3:1 - 3:10](
+        name: Name[3:1 - 3:3](
+            parts: array(
+                0: foo
+            )
+        )
+        args: array(
+            0: Arg[3:5 - 3:9](
+                value: Expr_ClassConstFetch[3:5 - 3:9](
+                    class: Name[3:5 - 3:7](
+                        parts: array(
+                            0: Bar
+                        )
+                    )
+                    name: Expr_Error[3:10 - 3:9](
+                    )
+                )
+                byRef: false
+                unpack: false
+            )
+        )
+    )
 )

test/code/prettyPrinter/commentsInCommaList.test

@@ -0,0 +1,53 @@
+Comments in arrays and function calls
+-----
+<?php
+
+$arr = [
+    // Foo
+    $foo,
+    // Bar
+    $bar,
+    // Discarded
+];
+[
+    // Foo
+    $foo,
+    ,
+    // Bar
+    $bar,
+] = $arr;
+foo(
+    // Foo
+    $foo,
+    // Bar
+    $bar
+);
+new Foo(
+    // Foo
+    $foo
+);
+-----
+!!php7
+$arr = [
+    // Foo
+    $foo,
+    // Bar
+    $bar,
+];
+[
+    // Foo
+    $foo,
+    ,
+    // Bar
+    $bar,
+] = $arr;
+foo(
+    // Foo
+    $foo,
+    // Bar
+    $bar
+);
+new Foo(
+    // Foo
+    $foo
+);