Move "kind" attribute handling out of lexer

Doesn't belong there and will cause issue with multiple assignments.

bin/php-parse

@@ -27,7 +27,7 @@ if (empty($files)) {
 }
 
 $lexer = new PhpParser\Lexer\Emulative(array('usedAttributes' => array(
-    'startLine', 'endLine', 'startFilePos', 'endFilePos', 'comments', 'kind'
+    'startLine', 'endLine', 'startFilePos', 'endFilePos', 'comments'
 )));
 $parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::PREFER_PHP7, $lexer);
 $dumper = new PhpParser\NodeDumper(['dumpComments' => true]);

grammar/php5.y

@@ -585,7 +585,8 @@ expr:
     | T_OBJECT_CAST expr                                    { $$ = Expr\Cast\Object_ [$2]; }
     | T_BOOL_CAST expr                                      { $$ = Expr\Cast\Bool_   [$2]; }
     | T_UNSET_CAST expr                                     { $$ = Expr\Cast\Unset_  [$2]; }
-    | T_EXIT exit_expr                                      { $$ = Expr\Exit_        [$2]; }
+    | T_EXIT exit_expr
+          { $attrs = attributes(); $attrs['kind'] = strtolower($1); $$ = new Expr\Exit_($2, $attrs); }
     | '@' expr                                              { $$ = Expr\ErrorSuppress[$2]; }
     | scalar                                                { $$ = $1; }
     | array_expr                                            { $$ = $1; }
@@ -735,7 +736,7 @@ ctor_arguments:
 ;
 
 common_scalar:
-      T_LNUMBER                                             { $$ = Scalar\LNumber[Scalar\LNumber::parse($1)]; }
+      T_LNUMBER                                             { $$ = Scalar\LNumber::fromString($1, attributes()); }
     | T_DNUMBER                                             { $$ = Scalar\DNumber[Scalar\DNumber::parse($1)]; }
     | T_CONSTANT_ENCAPSED_STRING                            { $$ = Scalar\String_[Scalar\String_::parse($1, false)]; }
     | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }

grammar/php7.y

@@ -576,7 +576,8 @@ expr:
     | T_OBJECT_CAST expr                                    { $$ = Expr\Cast\Object_ [$2]; }
     | T_BOOL_CAST expr                                      { $$ = Expr\Cast\Bool_   [$2]; }
     | T_UNSET_CAST expr                                     { $$ = Expr\Cast\Unset_  [$2]; }
-    | T_EXIT exit_expr                                      { $$ = Expr\Exit_        [$2]; }
+    | T_EXIT exit_expr
+          { $attrs = attributes(); $attrs['kind'] = strtolower($1); $$ = new Expr\Exit_($2, $attrs); }
     | '@' expr                                              { $$ = Expr\ErrorSuppress[$2]; }
     | scalar                                                { $$ = $1; }
     | '`' backticks_expr '`'                                { $$ = Expr\ShellExec[$2]; }
@@ -679,7 +680,7 @@ dereferencable_scalar:
 ;
 
 scalar:
-      T_LNUMBER                                             { $$ = Scalar\LNumber[Scalar\LNumber::parse($1)]; }
+      T_LNUMBER                                             { $$ = Scalar\LNumber::fromString($1, attributes()); }
     | T_DNUMBER                                             { $$ = Scalar\DNumber[Scalar\DNumber::parse($1)]; }
     | T_LINE                                                { $$ = Scalar\MagicConst\Line[]; }
     | T_FILE                                                { $$ = Scalar\MagicConst\File[]; }

lib/PhpParser/Lexer.php

@@ -23,9 +23,9 @@ class Lexer
      *
      * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
      *                       which is an array of attributes to add to the AST nodes. Possible
-     *                       attributes are: 'comments', 'kind', 'startLine', 'endLine',
+     *                       attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
-     *                       'startTokenPos', 'endTokenPos', 'startFilePos', 'endFilePos'. The
+     *                       'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
-     *                       option defaults to the first four. For more info see getNextToken() docs.
+     *                       first three. For more info see getNextToken() docs.
      */
     public function __construct(array $options = array()) {
         // map from internal tokens to PhpParser tokens
@@ -38,7 +38,7 @@ class Lexer
         // the usedAttributes member is a map of the used attribute names to a dummy
         // value (here "true")
         $options += array(
-            'usedAttributes' => array('comments', 'kind', 'startLine', 'endLine'),
+            'usedAttributes' => array('comments', 'startLine', 'endLine'),
         );
         $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
     }
@@ -116,8 +116,6 @@ class Lexer
      *  * 'comments'      => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
      *                       representing all comments that occurred between the previous
      *                       non-discarded token and the current one.
-     *  * 'kind'          => For some tokens, this will provide additional information about the
-     *                       formatting of the node (e.g. whether a numeric literal is binary/...)
      *  * 'startLine'     => Line in which the node starts.
      *  * 'endLine'       => Line in which the node ends.
      *  * 'startTokenPos' => Offset into the token array of the first token in the node.
@@ -204,22 +202,6 @@ class Lexer
                         $endAttributes['endFilePos'] = $this->filePos - 1;
                     }
 
-                    if (isset($this->usedAttributes['kind'])) {
-                        if ($token[0] === T_EXIT) {
-                            $startAttributes['kind'] = strtolower($value);
-                        } elseif ($token[0] === T_LNUMBER) {
-                            if ($value === '0' || $value[0] !== '0') {
-                                $startAttributes['kind'] = LNumber::KIND_DEC;
-                            } elseif ($value[1] === 'x' || $value[1] === 'X') {
-                                $startAttributes['kind'] = LNumber::KIND_HEX;
-                            } elseif ($value[1] === 'b' || $value[1] === 'B') {
-                                $startAttributes['kind'] = LNumber::KIND_BIN;
-                            } else {
-                                $startAttributes['kind'] = LNumber::KIND_OCT;
-                            }
-                        }
-                    }
-
                     return $this->tokenMap[$token[0]];
                 }
             }

lib/PhpParser/Node/Scalar/LNumber.php

@@ -30,6 +30,27 @@ class LNumber extends Scalar
         return array('value');
     }
 
+    /**
+     * Constructs an LNumber node from a string number literal.
+     *
+     * @param string $str        String number literal (decimal, octal, hex or binary)
+     * @param array  $attributes Additional attributes
+     *
+     * @return LNumber The constructed LNumber, including kind attribute
+     */
+    public static function fromString($str, array $attributes = array()) {
+        if ($str === '0' || $str[0] !== '0') {
+            $attributes['kind'] = LNumber::KIND_DEC;
+        } elseif ($str[1] === 'x' || $str[1] === 'X') {
+            $attributes['kind'] = LNumber::KIND_HEX;
+        } elseif ($str[1] === 'b' || $str[1] === 'B') {
+            $attributes['kind'] = LNumber::KIND_BIN;
+        } else {
+            $attributes['kind'] = LNumber::KIND_OCT;
+        }
+        return new self(self::parse($str), $attributes);
+    }
+
     /**
      * @internal
      *

lib/PhpParser/Parser/Php5.php

@@ -2352,7 +2352,7 @@ class Php5 extends \PhpParser\ParserAbstract
     }
 
     protected function reduceRule361() {
-         $this->semValue = new Expr\Exit_($this->semStack[$this->stackPos-(2-2)], $this->startAttributeStack[$this->stackPos-(2-1)] + $this->endAttributes);
+         $attrs = $this->startAttributeStack[$this->stackPos-(2-1)] + $this->endAttributes; $attrs['kind'] = strtolower($this->semStack[$this->stackPos-(2-1)]); $this->semValue = new Expr\Exit_($this->semStack[$this->stackPos-(2-2)], $attrs);
     }
 
     protected function reduceRule362() {
@@ -2600,7 +2600,7 @@ class Php5 extends \PhpParser\ParserAbstract
     }
 
     protected function reduceRule419() {
-         $this->semValue = new Scalar\LNumber(Scalar\LNumber::parse($this->semStack[$this->stackPos-(1-1)]), $this->startAttributeStack[$this->stackPos-(1-1)] + $this->endAttributes);
+         $this->semValue = Scalar\LNumber::fromString($this->semStack[$this->stackPos-(1-1)], $this->startAttributeStack[$this->stackPos-(1-1)] + $this->endAttributes);
     }
 
     protected function reduceRule420() {

lib/PhpParser/Parser/Php7.php

@@ -2202,7 +2202,7 @@ class Php7 extends \PhpParser\ParserAbstract
     }
 
     protected function reduceRule353() {
-         $this->semValue = new Expr\Exit_($this->semStack[$this->stackPos-(2-2)], $this->startAttributeStack[$this->stackPos-(2-1)] + $this->endAttributes);
+         $attrs = $this->startAttributeStack[$this->stackPos-(2-1)] + $this->endAttributes; $attrs['kind'] = strtolower($this->semStack[$this->stackPos-(2-1)]); $this->semValue = new Expr\Exit_($this->semStack[$this->stackPos-(2-2)], $attrs);
     }
 
     protected function reduceRule354() {
@@ -2376,7 +2376,7 @@ class Php7 extends \PhpParser\ParserAbstract
     }
 
     protected function reduceRule396() {
-         $this->semValue = new Scalar\LNumber(Scalar\LNumber::parse($this->semStack[$this->stackPos-(1-1)]), $this->startAttributeStack[$this->stackPos-(1-1)] + $this->endAttributes);
+         $this->semValue = Scalar\LNumber::fromString($this->semStack[$this->stackPos-(1-1)], $this->startAttributeStack[$this->stackPos-(1-1)] + $this->endAttributes);
     }
 
     protected function reduceRule397() {

test/PhpParser/Parser/MultipleTest.php

@@ -61,7 +61,7 @@ class MultipleTest extends ParserTest {
                 $this->getPrefer5(),
                 [
                     new Expr\Variable(
-                        new Expr\ArrayDimFetch(new Expr\Variable('a'), new LNumber(0))
+                        new Expr\ArrayDimFetch(new Expr\Variable('a'), LNumber::fromString('0'))
                     )
                 ]
             ],
@@ -71,7 +71,7 @@ class MultipleTest extends ParserTest {
                 $this->getPrefer7(),
                 [
                     new Expr\ArrayDimFetch(
-                        new Expr\Variable(new Expr\Variable('a')), new LNumber(0)
+                        new Expr\Variable(new Expr\Variable('a')), LNumber::fromString('0')
                     )
                 ]
             ],

test/PhpParser/Serializer/XMLTest.php

@@ -68,12 +68,12 @@ CODE;
         <attribute:startLine>
          <scalar:int>4</scalar:int>
         </attribute:startLine>
-        <attribute:kind>
-          <scalar:int>10</scalar:int>
-        </attribute:kind>
         <attribute:endLine>
          <scalar:int>4</scalar:int>
         </attribute:endLine>
+        <attribute:kind>
+          <scalar:int>10</scalar:int>
+        </attribute:kind>
         <subNode:value>
          <scalar:int>0</scalar:int>
         </subNode:value>