WIP

composer.json

@@ -14,6 +14,7 @@
     ],
     "require": {
         "php": ">=7.0",
+        "ext-json": "*",
         "ext-tokenizer": "*"
     },
     "require-dev": {

lib/PhpParser/FileContext.php

@@ -0,0 +1,8 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser;
+
+class FileContext {
+    /** @var Token[] */
+    public $tokens;
+}

lib/PhpParser/Lexer.php

@@ -227,23 +227,12 @@ class Lexer
      *
      * @return int Token id
      */
-    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
+    public function getNextToken(&$value = null, &$startAttributes = null) : int {
         $startAttributes = [];
-        $endAttributes   = [];
 
         while (1) {
             $token = $this->tokens[++$this->pos];
 
-            if ($this->attributeStartLineUsed) {
-                $startAttributes['startLine'] = $token->line;
-            }
-            if ($this->attributeStartTokenPosUsed) {
-                $startAttributes['startTokenPos'] = $this->pos;
-            }
-            if ($this->attributeStartFilePosUsed) {
-                $startAttributes['startFilePos'] = $token->filePos;
-            }
-
             $phpId = $token->id;
             $value = $token->value;
             if (!isset($this->dropTokens[$phpId])) {
@@ -254,16 +243,6 @@ class Lexer
                     $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
                 }
 
-                if ($this->attributeEndLineUsed) {
-                    $endAttributes['endLine'] = $token->line + substr_count($value, "\n");
-                }
-                if ($this->attributeEndTokenPosUsed) {
-                    $endAttributes['endTokenPos'] = $this->pos;
-                }
-                if ($this->attributeEndFilePosUsed) {
-                    $endAttributes['endFilePos'] = $token->filePos + \strlen($value) - 1;
-                }
-
                 return $id;
             }
 
@@ -281,19 +260,18 @@ class Lexer
     }
 
     /**
-     * Returns the token array for current code.
+     * Returns the token array for the current code.
      *
-     * The token array is in the same format as provided by the
+     * @return Token[]
-     * token_get_all() function and does not discard tokens (i.e.
-     * whitespace and comments are included). The token position
-     * attributes are against this token array.
-     *
-     * @return array Array of tokens in token_get_all() format
      */
     public function getTokens() : array {
         return $this->tokens;
     }
 
+    public function getTokenMap(): array {
+        return $this->tokenMap;
+    }
+
     /**
      * Handles __halt_compiler() by returning the text after it.
      *
@@ -324,6 +302,8 @@ class Lexer
      * The token map maps the PHP internal token identifiers
      * to the identifiers used by the Parser. Additionally it
      * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
+     * Whitespace and comment tokens are mapped to null, which indicates
+     * that they should be dropped.
      *
      * @return array The token map
      */
@@ -346,23 +326,16 @@ class Lexer
                 // T_CLOSE_TAG is equivalent to ';'
                 $tokenMap[$i] = ord(';');
             } elseif ('UNKNOWN' !== $name = token_name($i)) {
-                if ('T_HASHBANG' === $name) {
+                if (defined($name = Tokens::class . '::' . $name)) {
-                    // HHVM uses a special token for #! hashbang lines
-                    $tokenMap[$i] = Tokens::T_INLINE_HTML;
-                } elseif (defined($name = Tokens::class . '::' . $name)) {
                     // Other tokens can be mapped directly
                     $tokenMap[$i] = constant($name);
                 }
             }
         }
 
-        // HHVM uses a special token for numbers that overflow to double
+        $dropTokens = [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, self::T_BAD_CHARACTER];
-        if (defined('T_ONUMBER')) {
+        foreach ($dropTokens as $dropToken) {
-            $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
+            $tokenMap[$dropToken] = null;
-        }
-        // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
-        if (defined('T_COMPILER_HALT_OFFSET')) {
-            $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
         }
 
         return $tokenMap;

lib/PhpParser/NodeAbstract.php

@@ -4,8 +4,14 @@ namespace PhpParser;
 
 abstract class NodeAbstract implements Node, \JsonSerializable
 {
+    // TODO: Kill.
     protected $attributes;
 
+    /** @var FileContext|null  */
+    protected $context = null;
+    protected $startTokenPos = -1;
+    protected $endTokenPos = -1;
+
     /**
      * Creates a Node.
      *
@@ -15,35 +21,42 @@ abstract class NodeAbstract implements Node, \JsonSerializable
         $this->attributes = $attributes;
     }
 
+    public function setTokenContext(FileContext $context, int $firstToken, int $lastToken) {
+        $this->context = $context;
+        $this->startTokenPos = $firstToken;
+        $this->endTokenPos = $lastToken;
+    }
+
     /**
      * Gets line the node started in (alias of getStartLine).
      *
      * @return int Start line (or -1 if not available)
      */
     public function getLine() : int {
-        return $this->attributes['startLine'] ?? -1;
+        return $this->context->tokens[$this->startTokenPos]->line ?? -1;
     }
 
     /**
      * Gets line the node started in.
      *
-     * Requires the 'startLine' attribute to be enabled in the lexer (enabled by default).
-     *
      * @return int Start line (or -1 if not available)
      */
     public function getStartLine() : int {
-        return $this->attributes['startLine'] ?? -1;
+        return $this->context->tokens[$this->startTokenPos]->line ?? -1;
     }
 
     /**
      * Gets the line the node ended in.
      *
-     * Requires the 'endLine' attribute to be enabled in the lexer (enabled by default).
-     *
      * @return int End line (or -1 if not available)
      */
     public function getEndLine() : int {
-        return $this->attributes['endLine'] ?? -1;
+        if (!isset($this->context->tokens[$this->endTokenPos])) {
+            return -1;
+        }
+
+        $token = $this->context->tokens[$this->endTokenPos];
+        return $token->line + \substr_count($token, "\n");
     }
 
     /**
@@ -51,12 +64,10 @@ abstract class NodeAbstract implements Node, \JsonSerializable
      *
      * The offset is an index into the array returned by Lexer::getTokens().
      *
-     * Requires the 'startTokenPos' attribute to be enabled in the lexer (DISABLED by default).
-     *
      * @return int Token start position (or -1 if not available)
      */
     public function getStartTokenPos() : int {
-        return $this->attributes['startTokenPos'] ?? -1;
+        return $this->startTokenPos;
     }
 
     /**
@@ -64,34 +75,33 @@ abstract class NodeAbstract implements Node, \JsonSerializable
      *
      * The offset is an index into the array returned by Lexer::getTokens().
      *
-     * Requires the 'endTokenPos' attribute to be enabled in the lexer (DISABLED by default).
-     *
      * @return int Token end position (or -1 if not available)
      */
     public function getEndTokenPos() : int {
-        return $this->attributes['endTokenPos'] ?? -1;
+        return $this->endTokenPos;
     }
 
     /**
      * Gets the file offset of the first character that is part of this node.
      *
-     * Requires the 'startFilePos' attribute to be enabled in the lexer (DISABLED by default).
-     *
      * @return int File start position (or -1 if not available)
      */
     public function getStartFilePos() : int {
-        return $this->attributes['startFilePos'] ?? -1;
+        return $this->context->tokens[$this->startTokenPos]->filePos ?? -1;
     }
 
     /**
      * Gets the file offset of the last character that is part of this node.
      *
-     * Requires the 'endFilePos' attribute to be enabled in the lexer (DISABLED by default).
-     *
      * @return int File end position (or -1 if not available)
      */
     public function getEndFilePos() : int {
-        return $this->attributes['endFilePos'] ?? -1;
+        if (!isset($this->context->tokens[$this->endTokenPos])) {
+            return -1;
+        }
+
+        $token = $this->context->tokens[$this->endTokenPos];
+        return $token->filePos + \strlen($token->value) - 1;
     }
 
     /**