Update lexer implementations and tests

Start on lexer refactoring
WIP
2025-06-18 22:02:07 +02:00 · 2019-06-30 23:07:22 +02:00 · 2019-06-30 19:31:22 +02:00 · 2019-06-30 17:21:55 +02:00 · 2019-06-30 14:14:24 +02:00
14 changed files with 351 additions and 706 deletions
--- a/composer.json
+++ b/composer.json
@ -14,6 +14,7 @@
    ],
    "require": {
        "php": ">=7.0",
+        "ext-json": "*",
        "ext-tokenizer": "*"
    },
    "require-dev": {
--- a/lib/PhpParser/FileContext.php
+++ b/lib/PhpParser/FileContext.php
@ -0,0 +1,8 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser;
+
+class FileContext {
+    /** @var Token[] */
+    public $tokens;
+}
--- a/lib/PhpParser/Internal/TokenStream.php
+++ b/lib/PhpParser/Internal/TokenStream.php
@ -2,6 +2,8 @@

 namespace PhpParser\Internal;

+use PhpParser\Token;
+
 /**
 * Provides operations on token streams, for use by pretty printer.
 *
@ -9,7 +11,7 @@ namespace PhpParser\Internal;
 */
 class TokenStream
 {
-    /** @var array Tokens (in token_get_all format) */
+    /** @var Token[] */
    private $tokens;
    /** @var int[] Map from position to indentation */
    private $indentMap;
@ -17,7 +19,7 @@ class TokenStream
    /**
     * Create token stream instance.
     *
-     * @param array $tokens Tokens in token_get_all() format
+     * @param Token[] $tokens Tokens
     */
    public function __construct(array $tokens) {
        $this->tokens = $tokens;
@ -33,8 +35,8 @@ class TokenStream
     * @return bool
     */
    public function haveParens(int $startPos, int $endPos) : bool {
-        return $this->haveTokenImmediativelyBefore($startPos, '(')
-            && $this->haveTokenImmediatelyAfter($endPos, ')');
+        return $this->haveTokenImmediativelyBefore($startPos, \ord('('))
+            && $this->haveTokenImmediatelyAfter($endPos, \ord(')'));
    }

    /**
@ -46,8 +48,8 @@ class TokenStream
     * @return bool
     */
    public function haveBraces(int $startPos, int $endPos) : bool {
-        return $this->haveTokenImmediativelyBefore($startPos, '{')
-            && $this->haveTokenImmediatelyAfter($endPos, '}');
+        return $this->haveTokenImmediativelyBefore($startPos, \ord('{'))
+            && $this->haveTokenImmediatelyAfter($endPos, \ord('}'));
    }

    /**
@ -64,7 +66,7 @@ class TokenStream
        $tokens = $this->tokens;
        $pos--;
        for (; $pos >= 0; $pos--) {
-            $tokenType = $tokens[$pos][0];
+            $tokenType = $tokens[$pos]->id;
            if ($tokenType === $expectedTokenType) {
                return true;
            }
@ -90,7 +92,7 @@ class TokenStream
        $tokens = $this->tokens;
        $pos++;
        for (; $pos < \count($tokens); $pos++) {
-            $tokenType = $tokens[$pos][0];
+            $tokenType = $tokens[$pos]->id;
            if ($tokenType === $expectedTokenType) {
                return true;
            }
@ -110,7 +112,7 @@ class TokenStream
            return $pos;
        }

-        if ($tokens[$pos][0] !== $skipTokenType) {
+        if ($tokens[$pos]->id !== $skipTokenType) {
            // Shouldn't happen. The skip token MUST be there
            throw new \Exception('Encountered unexpected token');
        }
@ -127,7 +129,7 @@ class TokenStream
            return $pos;
        }

-        if ($tokens[$pos][0] !== $skipTokenType) {
+        if ($tokens[$pos]->id !== $skipTokenType) {
            // Shouldn't happen. The skip token MUST be there
            throw new \Exception('Encountered unexpected token');
        }
@ -145,7 +147,7 @@ class TokenStream
    public function skipLeftWhitespace(int $pos) {
        $tokens = $this->tokens;
        for (; $pos >= 0; $pos--) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
            if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                break;
            }
@ -162,7 +164,7 @@ class TokenStream
    public function skipRightWhitespace(int $pos) {
        $tokens = $this->tokens;
        for ($count = \count($tokens); $pos < $count; $pos++) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
            if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
                break;
            }
@ -173,7 +175,7 @@ class TokenStream
    public function findRight($pos, $findTokenType) {
        $tokens = $this->tokens;
        for ($count = \count($tokens); $pos < $count; $pos++) {
-            $type = $tokens[$pos][0];
+            $type = $tokens[$pos]->id;
            if ($type === $findTokenType) {
                return $pos;
            }
@ -206,23 +208,19 @@ class TokenStream
        $result = '';
        for ($pos = $from; $pos < $to; $pos++) {
            $token = $tokens[$pos];
-            if (\is_array($token)) {
-                $type = $token[0];
-                $content = $token[1];
-                if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
-                    $result .= $content;
-                } else {
-                    // TODO Handle non-space indentation
-                    if ($indent < 0) {
-                        $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
-                    } elseif ($indent > 0) {
-                        $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
-                    } else {
-                        $result .= $content;
-                    }
-                }
+            $type = $token->id;
+            $content = $token->value;
+            if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
+                $result .= $content;
            } else {
-                $result .= $token;
+                // TODO Handle non-space indentation
+                if ($indent < 0) {
+                    $result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
+                } elseif ($indent > 0) {
+                    $result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
+                } else {
+                    $result .= $content;
+                }
            }
        }
        return $result;
@ -239,8 +237,8 @@ class TokenStream
        foreach ($this->tokens as $token) {
            $indentMap[] = $indent;

-            if ($token[0] === \T_WHITESPACE) {
-                $content = $token[1];
+            if ($token->id === \T_WHITESPACE) {
+                $content = $token->value;
                $newlinePos = \strrpos($content, "\n");
                if (false !== $newlinePos) {
                    $indent = \strlen($content) - $newlinePos - 1;
--- a/lib/PhpParser/Lexer.php
+++ b/lib/PhpParser/Lexer.php
@ -6,100 +6,132 @@ use PhpParser\Parser\Tokens;

 class Lexer
 {
-    /* Token ID used for illegal characters part of the token stream. These are dropped by token_get_all(),
-     * but we restore them here to make sure that the tokens cover the full original text, and to prevent
-     * file positions from going out of sync. */
-    const T_BAD_CHARACTER = -1;
-
-    protected $code;
-    protected $tokens;
-    protected $pos;
-    protected $line;
-    protected $filePos;
-    protected $prevCloseTagHasNewline;
-
+    /** @var array Map from PHP tokens to PhpParser tokens. */
    protected $tokenMap;
-    protected $dropTokens;
-
-    private $attributeStartLineUsed;
-    private $attributeEndLineUsed;
-    private $attributeStartTokenPosUsed;
-    private $attributeEndTokenPosUsed;
-    private $attributeStartFilePosUsed;
-    private $attributeEndFilePosUsed;
-    private $attributeCommentsUsed;

    /**
     * Creates a Lexer.
     *
-     * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
-     *                       which is an array of attributes to add to the AST nodes. Possible
-     *                       attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
-     *                       'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
-     *                       first three. For more info see getNextToken() docs.
+     * @param array $options Options array. Currently unused.
     */
    public function __construct(array $options = []) {
-        // map from internal tokens to PhpParser tokens
        $this->tokenMap = $this->createTokenMap();
-
-        // map of tokens to drop while lexing (the map is only used for isset lookup,
-        // that's why the value is simply set to 1; the value is never actually used.)
-        $this->dropTokens = array_fill_keys(
-            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, self::T_BAD_CHARACTER], 1
-        );
-
-        $defaultAttributes = ['comments', 'startLine', 'endLine'];
-        $usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
-
-        // Create individual boolean properties to make these checks faster.
-        $this->attributeStartLineUsed = isset($usedAttributes['startLine']);
-        $this->attributeEndLineUsed = isset($usedAttributes['endLine']);
-        $this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
-        $this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
-        $this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
-        $this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
-        $this->attributeCommentsUsed = isset($usedAttributes['comments']);
    }

    /**
-     * Initializes the lexer for lexing the provided source code.
+     * Get tokens IDs that should be ignored by the parser.
+     *
+     * @return array
+     */
+    public function getIgnorableTokens(): array {
+        return [
+            Tokens::T_WHITESPACE,
+            Tokens::T_COMMENT,
+            Tokens::T_DOC_COMMENT,
+            Tokens::T_OPEN_TAG,
+            Tokens::T_BAD_CHARACTER,
+        ];
+    }
+
+    /**
+     * Get map for token canonicalization.
+     *
+     * @return array
+     */
+    public function getCanonicalizationMap(): array {
+        return [
+            Tokens::T_OPEN_TAG_WITH_ECHO => Tokens::T_ECHO,
+            Tokens::T_CLOSE_TAG => \ord(';'),
+        ];
+    }
+
+    /**
+     * Tokenizes the given PHP code into an array of Tokens.
     *
     * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
     * the getErrors() method.
     *
-     * @param string $code The source code to lex
+     * @param string $code The source code to tokenize
     * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
     *                                        ErrorHandler\Throwing
+     *
+     * @return Token[] Sequence of tokens
     */
-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+    public function tokenize(string $code, ErrorHandler $errorHandler = null) {
        if (null === $errorHandler) {
            $errorHandler = new ErrorHandler\Throwing();
        }

-        $this->code = $code; // keep the code around for __halt_compiler() handling
-        $this->pos  = -1;
-        $this->line =  1;
-        $this->filePos = 0;
-
-        // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
-        // This ensures proper composability, because having a newline is the "safe" assumption.
-        $this->prevCloseTagHasNewline = true;
-
        $scream = ini_set('xdebug.scream', '0');
-
        error_clear_last();
-        $this->tokens = @token_get_all($code);
-        $this->handleErrors($errorHandler);
-
+        $rawTokens = @token_get_all($code);
+        $checkForMissingTokens = null !== error_get_last();
        if (false !== $scream) {
            ini_set('xdebug.scream', $scream);
        }
+
+        $tokens = [];
+        $filePos = 0;
+        $line = 1;
+        foreach ($rawTokens as $rawToken) {
+            if (\is_array($rawToken)) {
+                $token = new Token($this->tokenMap[$rawToken[0]], $rawToken[1], $line, $filePos);
+            } elseif (\strlen($rawToken) == 2) {
+                // Bug in token_get_all() when lexing b".
+                $token = new Token(\ord('"'), $rawToken, $line, $filePos);
+            } else {
+                $token = new Token(\ord($rawToken), $rawToken, $line, $filePos);
+            }
+
+            $value = $token->value;
+            $tokenLen = \strlen($value);
+            if ($checkForMissingTokens && substr($code, $filePos, $tokenLen) !== $value) {
+                // Something is missing, must be an invalid character
+                $nextFilePos = strpos($code, $value, $filePos);
+                $badCharTokens = $this->handleInvalidCharacterRange(
+                    $code, $filePos, $nextFilePos, $line, $errorHandler);
+                $tokens = array_merge($tokens, $badCharTokens);
+                $filePos = (int) $nextFilePos;
+            }
+
+            $tokens[] = $token;
+            $filePos += $tokenLen;
+            $line += substr_count($value, "\n");
+        }
+
+        if ($filePos !== \strlen($code)) {
+            // Invalid characters at the end of the input
+            $badCharTokens = $this->handleInvalidCharacterRange(
+                $code, $filePos, \strlen($code), $line, $errorHandler);
+            $tokens = array_merge($tokens, $badCharTokens);
+        }
+
+        if (\count($tokens) > 0) {
+            // Check for unterminated comment
+            $lastToken = $tokens[\count($tokens) - 1];
+            if ($this->isUnterminatedComment($lastToken)) {
+                $errorHandler->handleError(new Error('Unterminated comment', [
+                    'startLine' => $line - substr_count($lastToken->value, "\n"),
+                    'endLine' => $line,
+                    'startFilePos' => $filePos - \strlen($lastToken->value),
+                    'endFilePos' => $filePos,
+                ]));
+            }
+        }
+
+        // Add an EOF sentinel token
+        // TODO: Should the value be an empty string instead?
+        $tokens[] = new Token(0, "\0", $line, \strlen($code));
+
+        return $tokens;
    }

-    private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
+    private function handleInvalidCharacterRange(
+        string $code, int $start, int $end, int $line, ErrorHandler $errorHandler
+    ) {
        $tokens = [];
        for ($i = $start; $i < $end; $i++) {
-            $chr = $this->code[$i];
+            $chr = $code[$i];
            if ($chr === "\0") {
                // PHP cuts error message after null byte, so need special case
                $errorMsg = 'Unexpected null byte';
@ -109,7 +141,7 @@ class Lexer
                );
            }

-            $tokens[] = [self::T_BAD_CHARACTER, $chr, $line];
+            $tokens[] = new Token(Tokens::T_BAD_CHARACTER, $chr, $line, $i);
            $errorHandler->handleError(new Error($errorMsg, [
                'startLine' => $line,
                'endLine' => $line,
@ -120,278 +152,24 @@ class Lexer
        return $tokens;
    }

-    /**
-     * Check whether comment token is unterminated.
-     *
-     * @return bool
-     */
-    private function isUnterminatedComment($token) : bool {
-        return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
-            && substr($token[1], 0, 2) === '/*'
-            && substr($token[1], -2) !== '*/';
+    private function isUnterminatedComment(Token $token): bool {
+        return ($token->id === \T_COMMENT || $token->id === \T_DOC_COMMENT)
+            && substr($token->value, 0, 2) === '/*'
+            && substr($token->value, -2) !== '*/';
    }

-    /**
-     * Check whether an error *may* have occurred during tokenization.
-     *
-     * @return bool
-     */
-    private function errorMayHaveOccurred() : bool {
-        if (defined('HHVM_VERSION')) {
-            // In HHVM token_get_all() does not throw warnings, so we need to conservatively
-            // assume that an error occurred
-            return true;
-        }
-
-        return null !== error_get_last();
-    }
-
-    protected function handleErrors(ErrorHandler $errorHandler) {
-        if (!$this->errorMayHaveOccurred()) {
-            return;
-        }
-
-        // PHP's error handling for token_get_all() is rather bad, so if we want detailed
-        // error information we need to compute it ourselves. Invalid character errors are
-        // detected by finding "gaps" in the token array. Unterminated comments are detected
-        // by checking if a trailing comment has a "*/" at the end.
-
-        $filePos = 0;
-        $line = 1;
-        $numTokens = \count($this->tokens);
-        for ($i = 0; $i < $numTokens; $i++) {
-            $token = $this->tokens[$i];
-            $tokenValue = \is_string($token) ? $token : $token[1];
-            $tokenLen = \strlen($tokenValue);
-
-            if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
-                // Something is missing, must be an invalid character
-                $nextFilePos = strpos($this->code, $tokenValue, $filePos);
-                $badCharTokens = $this->handleInvalidCharacterRange(
-                    $filePos, $nextFilePos, $line, $errorHandler);
-                $filePos = (int) $nextFilePos;
-
-                array_splice($this->tokens, $i, 0, $badCharTokens);
-                $numTokens += \count($badCharTokens);
-                $i += \count($badCharTokens);
-            }
-
-            $filePos += $tokenLen;
-            $line += substr_count($tokenValue, "\n");
-        }
-
-        if ($filePos !== \strlen($this->code)) {
-            if (substr($this->code, $filePos, 2) === '/*') {
-                // Unlike PHP, HHVM will drop unterminated comments entirely
-                $comment = substr($this->code, $filePos);
-                $errorHandler->handleError(new Error('Unterminated comment', [
-                    'startLine' => $line,
-                    'endLine' => $line + substr_count($comment, "\n"),
-                    'startFilePos' => $filePos,
-                    'endFilePos' => $filePos + \strlen($comment),
-                ]));
-
-                // Emulate the PHP behavior
-                $isDocComment = isset($comment[3]) && $comment[3] === '*';
-                $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
-            } else {
-                // Invalid characters at the end of the input
-                $badCharTokens = $this->handleInvalidCharacterRange(
-                    $filePos, \strlen($this->code), $line, $errorHandler);
-                $this->tokens = array_merge($this->tokens, $badCharTokens);
-            }
-            return;
-        }
-
-        if (count($this->tokens) > 0) {
-            // Check for unterminated comment
-            $lastToken = $this->tokens[count($this->tokens) - 1];
-            if ($this->isUnterminatedComment($lastToken)) {
-                $errorHandler->handleError(new Error('Unterminated comment', [
-                    'startLine' => $line - substr_count($lastToken[1], "\n"),
-                    'endLine' => $line,
-                    'startFilePos' => $filePos - \strlen($lastToken[1]),
-                    'endFilePos' => $filePos,
-                ]));
-            }
-        }
-    }
-
-    /**
-     * Fetches the next token.
-     *
-     * The available attributes are determined by the 'usedAttributes' option, which can
-     * be specified in the constructor. The following attributes are supported:
-     *
-     *  * 'comments'      => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
-     *                       representing all comments that occurred between the previous
-     *                       non-discarded token and the current one.
-     *  * 'startLine'     => Line in which the node starts.
-     *  * 'endLine'       => Line in which the node ends.
-     *  * 'startTokenPos' => Offset into the token array of the first token in the node.
-     *  * 'endTokenPos'   => Offset into the token array of the last token in the node.
-     *  * 'startFilePos'  => Offset into the code string of the first character that is part of the node.
-     *  * 'endFilePos'    => Offset into the code string of the last character that is part of the node.
-     *
-     * @param mixed $value           Variable to store token content in
-     * @param mixed $startAttributes Variable to store start attributes in
-     * @param mixed $endAttributes   Variable to store end attributes in
-     *
-     * @return int Token id
-     */
-    public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
-        $startAttributes = [];
-        $endAttributes   = [];
-
-        while (1) {
-            if (isset($this->tokens[++$this->pos])) {
-                $token = $this->tokens[$this->pos];
-            } else {
-                // EOF token with ID 0
-                $token = "\0";
-            }
-
-            if ($this->attributeStartLineUsed) {
-                $startAttributes['startLine'] = $this->line;
-            }
-            if ($this->attributeStartTokenPosUsed) {
-                $startAttributes['startTokenPos'] = $this->pos;
-            }
-            if ($this->attributeStartFilePosUsed) {
-                $startAttributes['startFilePos'] = $this->filePos;
-            }
-
-            if (\is_string($token)) {
-                $value = $token;
-                if (isset($token[1])) {
-                    // bug in token_get_all
-                    $this->filePos += 2;
-                    $id = ord('"');
-                } else {
-                    $this->filePos += 1;
-                    $id = ord($token);
-                }
-            } elseif (!isset($this->dropTokens[$token[0]])) {
-                $value = $token[1];
-                $id = $this->tokenMap[$token[0]];
-                if (\T_CLOSE_TAG === $token[0]) {
-                    $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
-                } elseif (\T_INLINE_HTML === $token[0]) {
-                    $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
-                }
-
-                $this->line += substr_count($value, "\n");
-                $this->filePos += \strlen($value);
-            } else {
-                if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
-                    if ($this->attributeCommentsUsed) {
-                        $comment = \T_DOC_COMMENT === $token[0]
-                            ? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
-                            : new Comment($token[1], $this->line, $this->filePos, $this->pos);
-                        $startAttributes['comments'][] = $comment;
-                    }
-                }
-
-                $this->line += substr_count($token[1], "\n");
-                $this->filePos += \strlen($token[1]);
-                continue;
-            }
-
-            if ($this->attributeEndLineUsed) {
-                $endAttributes['endLine'] = $this->line;
-            }
-            if ($this->attributeEndTokenPosUsed) {
-                $endAttributes['endTokenPos'] = $this->pos;
-            }
-            if ($this->attributeEndFilePosUsed) {
-                $endAttributes['endFilePos'] = $this->filePos - 1;
-            }
-
-            return $id;
-        }
-
-        throw new \RuntimeException('Reached end of lexer loop');
-    }
-
-    /**
-     * Returns the token array for current code.
-     *
-     * The token array is in the same format as provided by the
-     * token_get_all() function and does not discard tokens (i.e.
-     * whitespace and comments are included). The token position
-     * attributes are against this token array.
-     *
-     * @return array Array of tokens in token_get_all() format
-     */
-    public function getTokens() : array {
-        return $this->tokens;
-    }
-
-    /**
-     * Handles __halt_compiler() by returning the text after it.
-     *
-     * @return string Remaining text
-     */
-    public function handleHaltCompiler() : string {
-        // text after T_HALT_COMPILER, still including ();
-        $textAfter = substr($this->code, $this->filePos);
-
-        // ensure that it is followed by ();
-        // this simplifies the situation, by not allowing any comments
-        // in between of the tokens.
-        if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
-            throw new Error('__HALT_COMPILER must be followed by "();"');
-        }
-
-        // prevent the lexer from returning any further tokens
-        $this->pos = count($this->tokens);
-
-        // return with (); removed
-        return substr($textAfter, strlen($matches[0]));
-    }
-
-    /**
-     * Creates the token map.
-     *
-     * The token map maps the PHP internal token identifiers
-     * to the identifiers used by the Parser. Additionally it
-     * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
-     *
-     * @return array The token map
-     */
-    protected function createTokenMap() : array {
+    private function createTokenMap(): array {
        $tokenMap = [];

-        // 256 is the minimum possible token number, as everything below
-        // it is an ASCII value
        for ($i = 256; $i < 1000; ++$i) {
-            if (\T_DOUBLE_COLON === $i) {
-                // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
-                $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
-            } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
-                // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
-                $tokenMap[$i] = Tokens::T_ECHO;
-            } elseif(\T_CLOSE_TAG === $i) {
-                // T_CLOSE_TAG is equivalent to ';'
-                $tokenMap[$i] = ord(';');
-            } elseif ('UNKNOWN' !== $name = token_name($i)) {
-                if ('T_HASHBANG' === $name) {
-                    // HHVM uses a special token for #! hashbang lines
-                    $tokenMap[$i] = Tokens::T_INLINE_HTML;
-                } elseif (defined($name = Tokens::class . '::' . $name)) {
-                    // Other tokens can be mapped directly
-                    $tokenMap[$i] = constant($name);
-                }
+            $name = token_name($i);
+            if ('UNKNOWN' === $name) {
+                continue;
+            }
+            $constName = Tokens::class . '::' . $name;
+            if (defined($constName)) {
+                $tokenMap[$i] = constant($constName);
            }
-        }
-
-        // HHVM uses a special token for numbers that overflow to double
-        if (defined('T_ONUMBER')) {
-            $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
-        }
-        // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
-        if (defined('T_COMPILER_HALT_OFFSET')) {
-            $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
        }

        return $tokenMap;
--- a/lib/PhpParser/Lexer/Emulative.php
+++ b/lib/PhpParser/Lexer/Emulative.php
@ -16,9 +16,6 @@ class Emulative extends Lexer
    const PHP_7_3 = '7.3.0dev';
    const PHP_7_4 = '7.4.0dev';

-    const T_COALESCE_EQUAL = 1007;
-    const T_FN = 1008;
-
    const FLEXIBLE_DOC_STRING_REGEX = <<<'REGEX'
 /<<<[ \t]*(['"]?)([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)\1\r?\n
 (?:.*\r?\n)*?
@ -41,26 +38,22 @@ REGEX;
        $this->tokenEmulators[] = new FnTokenEmulator();
        $this->tokenEmulators[] = new CoaleseEqualTokenEmulator();
        $this->tokenEmulators[] = new NumericLiteralSeparatorEmulator();
-
-        $this->tokenMap[self::T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
-        $this->tokenMap[self::T_FN] = Tokens::T_FN;
    }

-    public function startLexing(string $code, ErrorHandler $errorHandler = null) {
+    public function tokenize(string $code, ErrorHandler $errorHandler = null) {
        $this->patches = [];

        if ($this->isEmulationNeeded($code) === false) {
            // Nothing to emulate, yay
-            parent::startLexing($code, $errorHandler);
-            return;
+            return parent::tokenize($code, $errorHandler);
        }

        $collector = new ErrorHandler\Collecting();

        // 1. emulation of heredoc and nowdoc new syntax
        $preparedCode = $this->processHeredocNowdoc($code);
-        parent::startLexing($preparedCode, $collector);
-        $this->fixupTokens();
+        $tokens = parent::tokenize($preparedCode, $collector);
+        $tokens = $this->fixupTokens($tokens);

        $errors = $collector->getErrors();
        if (!empty($errors)) {
@ -73,9 +66,11 @@ REGEX;
        // add token emulation
        foreach ($this->tokenEmulators as $emulativeToken) {
            if ($emulativeToken->isEmulationNeeded($code)) {
-                $this->tokens = $emulativeToken->emulate($code, $this->tokens);
+                $tokens = $emulativeToken->emulate($code, $tokens);
            }
        }
+
+        return $tokens;
    }

    private function isHeredocNowdocEmulationNeeded(string $code): bool
@ -144,10 +139,10 @@ REGEX;
        return $this->isHeredocNowdocEmulationNeeded($code);
    }

-    private function fixupTokens()
+    private function fixupTokens(array $tokens): array
    {
        if (\count($this->patches) === 0) {
-            return;
+            return $tokens;
        }

        // Load first patch
@ -157,35 +152,29 @@ REGEX;

        // We use a manual loop over the tokens, because we modify the array on the fly
        $pos = 0;
-        for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
-            $token = $this->tokens[$i];
-            if (\is_string($token)) {
-                // We assume that patches don't apply to string tokens
-                $pos += \strlen($token);
-                continue;
-            }
-
-            $len = \strlen($token[1]);
+        for ($i = 0, $c = \count($tokens); $i < $c; $i++) {
+            $token = $tokens[$i];
+            $len = \strlen($token->value);
            $posDelta = 0;
            while ($patchPos >= $pos && $patchPos < $pos + $len) {
                $patchTextLen = \strlen($patchText);
                if ($patchType === 'remove') {
                    if ($patchPos === $pos && $patchTextLen === $len) {
                        // Remove token entirely
-                        array_splice($this->tokens, $i, 1, []);
+                        array_splice($tokens, $i, 1, []);
                        $i--;
                        $c--;
                    } else {
                        // Remove from token string
-                        $this->tokens[$i][1] = substr_replace(
-                            $token[1], '', $patchPos - $pos + $posDelta, $patchTextLen
+                        $tokens[$i]->value = substr_replace(
+                            $token->value, '', $patchPos - $pos + $posDelta, $patchTextLen
                        );
                        $posDelta -= $patchTextLen;
                    }
                } elseif ($patchType === 'add') {
                    // Insert into the token string
-                    $this->tokens[$i][1] = substr_replace(
-                        $token[1], $patchText, $patchPos - $pos + $posDelta, 0
+                    $tokens[$i]->value = substr_replace(
+                        $token->value, $patchText, $patchPos - $pos + $posDelta, 0
                    );
                    $posDelta += $patchTextLen;
                } else {
@ -196,21 +185,30 @@ REGEX;
                $patchIdx++;
                if ($patchIdx >= \count($this->patches)) {
                    // No more patches, we're done
-                    return;
+                    break 2;
                }

                list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];

                // Multiple patches may apply to the same token. Reload the current one to check
                // If the new patch applies
-                $token = $this->tokens[$i];
+                $token = $tokens[$i];
            }

            $pos += $len;
        }

-        // A patch did not apply
-        assert(false);
+        // To retain a minimum amount of sanity, recompute lines and offsets in a separate loop.
+        $pos = 0;
+        $line = 1;
+        foreach ($tokens as $token) {
+            $token->filePos = $pos;
+            $token->line = $line;
+            $pos += \strlen($token->value);
+            $line += \substr_count($token->value, "\n");
+        }
+
+        return $tokens;
    }

    /**
--- a/lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php
+++ b/lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php
@ -3,6 +3,8 @@
 namespace PhpParser\Lexer\TokenEmulator;

 use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;

 final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
 {
@ -20,20 +22,17 @@ final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
    {
        // We need to manually iterate and manage a count because we'll change
        // the tokens array on the way
-        $line = 1;
        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
            if (isset($tokens[$i + 1])) {
-                if ($tokens[$i][0] === T_COALESCE && $tokens[$i + 1] === '=') {
+                $token = $tokens[$i];
+                if ($token->id === Tokens::T_COALESCE && $tokens[$i + 1]->value === '=') {
                    array_splice($tokens, $i, 2, [
-                        [Emulative::T_COALESCE_EQUAL, '??=', $line]
+                        new Token(Tokens::T_COALESCE_EQUAL, '??=', $token->line, $token->filePos),
                    ]);
                    $c--;
                    continue;
                }
            }
-            if (\is_array($tokens[$i])) {
-                $line += substr_count($tokens[$i][1], "\n");
-            }
        }

        return $tokens;
--- a/lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php
+++ b/lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php
@ -3,6 +3,8 @@
 namespace PhpParser\Lexer\TokenEmulator;

 use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;

 final class FnTokenEmulator implements TokenEmulatorInterface
 {
@ -21,13 +23,14 @@ final class FnTokenEmulator implements TokenEmulatorInterface
        // We need to manually iterate and manage a count because we'll change
        // the tokens array on the way
        foreach ($tokens as $i => $token) {
-            if ($token[0] === T_STRING && $token[1] === 'fn') {
+            if ($token->id === Tokens::T_STRING && $token->value === 'fn') {
                $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $i);
-                if ($previousNonSpaceToken !== null && $previousNonSpaceToken[0] === T_OBJECT_OPERATOR) {
+                if ($previousNonSpaceToken !== null
+                        && $previousNonSpaceToken->id === Tokens::T_OBJECT_OPERATOR) {
                    continue;
                }

-                $tokens[$i][0] = Emulative::T_FN;
+                $token->id = Tokens::T_FN;
            }
        }

@ -35,13 +38,13 @@ final class FnTokenEmulator implements TokenEmulatorInterface
    }

    /**
-     * @param mixed[] $tokens
-     * @return mixed[]|null
+     * @param Token[] $tokens
+     * @return Token|null
     */
    private function getPreviousNonSpaceToken(array $tokens, int $start)
    {
        for ($i = $start - 1; $i >= 0; --$i) {
-            if ($tokens[$i][0] === T_WHITESPACE) {
+            if ($tokens[$i]->id === Tokens::T_WHITESPACE) {
                continue;
            }

--- a/lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php
+++ b/lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php
@ -3,6 +3,8 @@
 namespace PhpParser\Lexer\TokenEmulator;

 use PhpParser\Lexer\Emulative;
+use PhpParser\Parser\Tokens;
+use PhpParser\Token;

 final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
 {
@ -24,47 +26,48 @@ final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
        return preg_match('~[0-9a-f]_[0-9a-f]~i', $code) !== false;
    }

+    /**
+     * @param Token[] $tokens
+     * @return Token[]
+     */
    public function emulate(string $code, array $tokens): array
    {
        // We need to manually iterate and manage a count because we'll change
        // the tokens array on the way
-        $codeOffset = 0;
        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
            $token = $tokens[$i];
-            $tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
+            $tokenLen = \strlen($token->value);

-            if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
-                $codeOffset += $tokenLen;
+            if ($token->id !== Tokens::T_LNUMBER && $token->id !== Tokens::T_DNUMBER) {
                continue;
            }

-            $res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
+            $res = preg_match(self::NUMBER, $code, $matches, 0, $token->filePos);
            assert($res, "No number at number token position");

            $match = $matches[0];
            $matchLen = \strlen($match);
            if ($matchLen === $tokenLen) {
                // Original token already holds the full number.
-                $codeOffset += $tokenLen;
                continue;
            }

            $tokenKind = $this->resolveIntegerOrFloatToken($match);
-            $newTokens = [[$tokenKind, $match, $token[2]]];
+            $newTokens = [new Token($tokenKind, $match, $token->line, $token->filePos)];

            $numTokens = 1;
            $len = $tokenLen;
            while ($matchLen > $len) {
                $nextToken = $tokens[$i + $numTokens];
-                $nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
-                $nextTokenLen = \strlen($nextTokenText);
+                $nextTokenLen = \strlen($nextToken->value);

                $numTokens++;
                if ($matchLen < $len + $nextTokenLen) {
                    // Split trailing characters into a partial token.
-                    assert(is_array($nextToken), "Partial token should be an array token");
-                    $partialText = substr($nextTokenText, $matchLen - $len);
-                    $newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
+                    $partialText = substr($nextToken->value, $matchLen - $len);
+                    $newTokens[] = new Token(
+                        $nextToken->id, $partialText, $nextToken->line, $nextToken->filePos
+                    );
                    break;
                }

@ -73,7 +76,6 @@ final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface

            array_splice($tokens, $i, $numTokens, $newTokens);
            $c -= $numTokens - \count($newTokens);
-            $codeOffset += $matchLen;
        }

        return $tokens;
@ -93,6 +95,6 @@ final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
            $num = +$str;
        }

-        return is_float($num) ? T_DNUMBER : T_LNUMBER;
+        return is_float($num) ? Tokens::T_DNUMBER : Tokens::T_LNUMBER;
    }
 }
--- a/lib/PhpParser/Lexer/TokenEmulator/TokenEmulatorInterface.php
+++ b/lib/PhpParser/Lexer/TokenEmulator/TokenEmulatorInterface.php
@ -2,13 +2,16 @@

 namespace PhpParser\Lexer\TokenEmulator;

+use PhpParser\Token;
+
 /** @internal */
 interface TokenEmulatorInterface
 {
    public function isEmulationNeeded(string $code): bool;

    /**
-     * @return array Modified Tokens
+     * @param Token[] $tokens
+     * @return Token[]
     */
    public function emulate(string $code, array $tokens): array;
 }
--- a/lib/PhpParser/NodeAbstract.php
+++ b/lib/PhpParser/NodeAbstract.php
@ -4,8 +4,14 @@ namespace PhpParser;

 abstract class NodeAbstract implements Node, \JsonSerializable
 {
+    // TODO: Kill.
    protected $attributes;

+    /** @var FileContext|null  */
+    protected $context = null;
+    protected $startTokenPos = -1;
+    protected $endTokenPos = -1;
+
    /**
     * Creates a Node.
     *
@ -15,35 +21,42 @@ abstract class NodeAbstract implements Node, \JsonSerializable
        $this->attributes = $attributes;
    }

+    public function setTokenContext(FileContext $context, int $firstToken, int $lastToken) {
+        $this->context = $context;
+        $this->startTokenPos = $firstToken;
+        $this->endTokenPos = $lastToken;
+    }
+
    /**
     * Gets line the node started in (alias of getStartLine).
     *
     * @return int Start line (or -1 if not available)
     */
    public function getLine() : int {
-        return $this->attributes['startLine'] ?? -1;
+        return $this->context->tokens[$this->startTokenPos]->line ?? -1;
    }

    /**
     * Gets line the node started in.
     *
-     * Requires the 'startLine' attribute to be enabled in the lexer (enabled by default).
-     *
     * @return int Start line (or -1 if not available)
     */
    public function getStartLine() : int {
-        return $this->attributes['startLine'] ?? -1;
+        return $this->context->tokens[$this->startTokenPos]->line ?? -1;
    }

    /**
     * Gets the line the node ended in.
     *
-     * Requires the 'endLine' attribute to be enabled in the lexer (enabled by default).
-     *
     * @return int End line (or -1 if not available)
     */
    public function getEndLine() : int {
-        return $this->attributes['endLine'] ?? -1;
+        if (!isset($this->context->tokens[$this->endTokenPos])) {
+            return -1;
+        }
+
+        $token = $this->context->tokens[$this->endTokenPos];
+        return $token->line + \substr_count($token, "\n");
    }

    /**
@ -51,12 +64,10 @@ abstract class NodeAbstract implements Node, \JsonSerializable
     *
     * The offset is an index into the array returned by Lexer::getTokens().
     *
-     * Requires the 'startTokenPos' attribute to be enabled in the lexer (DISABLED by default).
-     *
     * @return int Token start position (or -1 if not available)
     */
    public function getStartTokenPos() : int {
-        return $this->attributes['startTokenPos'] ?? -1;
+        return $this->startTokenPos;
    }

    /**
@ -64,34 +75,33 @@ abstract class NodeAbstract implements Node, \JsonSerializable
     *
     * The offset is an index into the array returned by Lexer::getTokens().
     *
-     * Requires the 'endTokenPos' attribute to be enabled in the lexer (DISABLED by default).
-     *
     * @return int Token end position (or -1 if not available)
     */
    public function getEndTokenPos() : int {
-        return $this->attributes['endTokenPos'] ?? -1;
+        return $this->endTokenPos;
    }

    /**
     * Gets the file offset of the first character that is part of this node.
     *
-     * Requires the 'startFilePos' attribute to be enabled in the lexer (DISABLED by default).
-     *
     * @return int File start position (or -1 if not available)
     */
    public function getStartFilePos() : int {
-        return $this->attributes['startFilePos'] ?? -1;
+        return $this->context->tokens[$this->startTokenPos]->filePos ?? -1;
    }

    /**
     * Gets the file offset of the last character that is part of this node.
     *
-     * Requires the 'endFilePos' attribute to be enabled in the lexer (DISABLED by default).
-     *
     * @return int File end position (or -1 if not available)
     */
    public function getEndFilePos() : int {
-        return $this->attributes['endFilePos'] ?? -1;
+        if (!isset($this->context->tokens[$this->endTokenPos])) {
+            return -1;
+        }
+
+        $token = $this->context->tokens[$this->endTokenPos];
+        return $token->filePos + \strlen($token->value) - 1;
    }

    /**
--- a/lib/PhpParser/PrettyPrinterAbstract.php
+++ b/lib/PhpParser/PrettyPrinterAbstract.php
@ -491,7 +491,7 @@ abstract class PrettyPrinterAbstract
        $pos = 0;
        $result = $this->pArray($stmts, $origStmts, $pos, 0, 'File', 'stmts', null);
        if (null !== $result) {
-            $result .= $this->origTokens->getTokenCode($pos, count($origTokens), 0);
+            $result .= $this->origTokens->getTokenCode($pos, count($origTokens) - 1, 0);
        } else {
            // Fallback
            // TODO Add <?php properly
@ -1213,8 +1213,8 @@ abstract class PrettyPrinterAbstract
        $stripLeft = ['left' => \T_WHITESPACE];
        $stripRight = ['right' => \T_WHITESPACE];
        $stripDoubleArrow = ['right' => \T_DOUBLE_ARROW];
-        $stripColon = ['left' => ':'];
-        $stripEquals = ['left' => '='];
+        $stripColon = ['left' => \ord(':')];
+        $stripEquals = ['left' => \ord('=')];
        $this->removalMap = [
            'Expr_ArrayDimFetch->dim' => $stripBoth,
            'Expr_ArrayItem->key' => $stripDoubleArrow,
@ -1254,22 +1254,22 @@ abstract class PrettyPrinterAbstract
        // TODO: "yield" where both key and value are inserted doesn't work
        // [$find, $beforeToken, $extraLeft, $extraRight]
        $this->insertionMap = [
-            'Expr_ArrayDimFetch->dim' => ['[', false, null, null],
+            'Expr_ArrayDimFetch->dim' => [\ord('['), false, null, null],
            'Expr_ArrayItem->key' => [null, false, null, ' => '],
-            'Expr_ArrowFunction->returnType' => [')', false, ' : ', null],
-            'Expr_Closure->returnType' => [')', false, ' : ', null],
-            'Expr_Ternary->if' => ['?', false, ' ', ' '],
+            'Expr_ArrowFunction->returnType' => [\ord(')'), false, ' : ', null],
+            'Expr_Closure->returnType' => [\ord(')'), false, ' : ', null],
+            'Expr_Ternary->if' => [\ord('?'), false, ' ', ' '],
            'Expr_Yield->key' => [\T_YIELD, false, null, ' => '],
            'Expr_Yield->value' => [\T_YIELD, false, ' ', null],
            'Param->type' => [null, false, null, ' '],
            'Param->default' => [null, false, ' = ', null],
            'Stmt_Break->num' => [\T_BREAK, false, ' ', null],
-            'Stmt_ClassMethod->returnType' => [')', false, ' : ', null],
+            'Stmt_ClassMethod->returnType' => [\ord(')'), false, ' : ', null],
            'Stmt_Class->extends' => [null, false, ' extends ', null],
            'Expr_PrintableNewAnonClass->extends' => [null, ' extends ', null],
            'Stmt_Continue->num' => [\T_CONTINUE, false, ' ', null],
            'Stmt_Foreach->keyVar' => [\T_AS, false, null, ' => '],
-            'Stmt_Function->returnType' => [')', false, ' : ', null],
+            'Stmt_Function->returnType' => [\ord(')'), false, ' : ', null],
            'Stmt_If->else' => [null, false, ' ', null],
            'Stmt_Namespace->name' => [\T_NAMESPACE, false, ' ', null],
            'Stmt_Property->type' => [\T_VARIABLE, true, null, ' '],
@ -1367,19 +1367,19 @@ abstract class PrettyPrinterAbstract

        // [$find, $extraLeft, $extraRight]
        $this->emptyListInsertionMap = [
-            'Expr_ArrowFunction->params' => ['(', '', ''],
-            'Expr_Closure->uses' => [')', ' use(', ')'],
-            'Expr_Closure->params' => ['(', '', ''],
-            'Expr_FuncCall->args' => ['(', '', ''],
-            'Expr_MethodCall->args' => ['(', '', ''],
-            'Expr_New->args' => ['(', '', ''],
-            'Expr_PrintableNewAnonClass->args' => ['(', '', ''],
+            'Expr_ArrowFunction->params' => [\ord('('), '', ''],
+            'Expr_Closure->uses' => [\ord(')'), ' use(', ')'],
+            'Expr_Closure->params' => [\ord('('), '', ''],
+            'Expr_FuncCall->args' => [\ord('('), '', ''],
+            'Expr_MethodCall->args' => [\ord('('), '', ''],
+            'Expr_New->args' => [\ord('('), '', ''],
+            'Expr_PrintableNewAnonClass->args' => [\ord('('), '', ''],
            'Expr_PrintableNewAnonClass->implements' => [null, ' implements ', ''],
-            'Expr_StaticCall->args' => ['(', '', ''],
+            'Expr_StaticCall->args' => [\ord('('), '', ''],
            'Stmt_Class->implements' => [null, ' implements ', ''],
-            'Stmt_ClassMethod->params' => ['(', '', ''],
+            'Stmt_ClassMethod->params' => [\ord('('), '', ''],
            'Stmt_Interface->extends' => [null, ' extends ', ''],
-            'Stmt_Function->params' => ['(', '', ''],
+            'Stmt_Function->params' => [\ord('('), '', ''],

            /* These cannot be empty to start with:
             * Expr_Isset->vars
--- a/lib/PhpParser/Token.php
+++ b/lib/PhpParser/Token.php
@ -0,0 +1,21 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser;
+
+class Token {
+    /** @var int Token id (a PhpParser\Parser\Tokens::T_* constant) */
+    public $id; // TODO: Move this to PhpParser\Tokens.
+    /** @var string Textual value of the token */
+    public $value;
+    /** @var int Start line number of the token */
+    public $line;
+    /** @var int Offset of the token in the source code */
+    public $filePos;
+
+    public function __construct(int $id, string $value, int $line, int $filePos) {
+        $this->id = $id;
+        $this->value = $value;
+        $this->line = $line;
+        $this->filePos = $filePos;
+    }
+}
--- a/test/PhpParser/Lexer/EmulativeTest.php
+++ b/test/PhpParser/Lexer/EmulativeTest.php
@ -5,6 +5,7 @@ namespace PhpParser\Lexer;
 use PhpParser\ErrorHandler;
 use PhpParser\LexerTest;
 use PhpParser\Parser\Tokens;
+use PhpParser\Token;

 class EmulativeTest extends LexerTest
 {
@ -17,10 +18,15 @@ class EmulativeTest extends LexerTest
     */
    public function testReplaceKeywords($keyword, $expectedToken) {
        $lexer = $this->getLexer();
-        $lexer->startLexing('<?php ' . $keyword);
-
-        $this->assertSame($expectedToken, $lexer->getNextToken());
-        $this->assertSame(0, $lexer->getNextToken());
+        $tokens = $lexer->tokenize('<?php ' . $keyword);
+        $this->assertEquals(
+            [
+                new Token(Tokens::T_OPEN_TAG, '<?php ', 1, 0),
+                new Token($expectedToken, $keyword, 1, 6),
+                new Token(0, "\0", 1, 6 + strlen($keyword)),
+            ],
+            $tokens
+        );
    }

    /**
@ -28,11 +34,16 @@ class EmulativeTest extends LexerTest
     */
    public function testNoReplaceKeywordsAfterObjectOperator(string $keyword) {
        $lexer = $this->getLexer();
-        $lexer->startLexing('<?php ->' . $keyword);
-
-        $this->assertSame(Tokens::T_OBJECT_OPERATOR, $lexer->getNextToken());
-        $this->assertSame(Tokens::T_STRING, $lexer->getNextToken());
-        $this->assertSame(0, $lexer->getNextToken());
+        $tokens = $lexer->tokenize('<?php ->' . $keyword);
+        $this->assertEquals(
+            [
+                new Token(Tokens::T_OPEN_TAG, '<?php ', 1, 0),
+                new Token(Tokens::T_OBJECT_OPERATOR, '->', 1, 6),
+                new Token(Tokens::T_STRING, $keyword, 1, 8),
+                new Token(0, "\0", 1, 8 + strlen($keyword)),
+            ],
+            $tokens
+        );
    }

    /**
@ -40,11 +51,17 @@ class EmulativeTest extends LexerTest
     */
    public function testNoReplaceKeywordsAfterObjectOperatorWithSpaces(string $keyword) {
        $lexer = $this->getLexer();
-        $lexer->startLexing('<?php ->    ' . $keyword);
-
-        $this->assertSame(Tokens::T_OBJECT_OPERATOR, $lexer->getNextToken());
-        $this->assertSame(Tokens::T_STRING, $lexer->getNextToken());
-        $this->assertSame(0, $lexer->getNextToken());
+        $tokens = $lexer->tokenize('<?php -> ' . $keyword);
+        $this->assertEquals(
+            [
+                new Token(Tokens::T_OPEN_TAG, '<?php ', 1, 0),
+                new Token(Tokens::T_OBJECT_OPERATOR, '->', 1, 6),
+                new Token(Tokens::T_WHITESPACE, ' ', 1, 8),
+                new Token(Tokens::T_STRING, $keyword, 1, 9),
+                new Token(0, "\0", 1, 9 + strlen($keyword)),
+            ],
+            $tokens
+        );
    }

    public function provideTestReplaceKeywords() {
@ -75,12 +92,12 @@ class EmulativeTest extends LexerTest
     */
    public function testLexNewFeatures($code, array $expectedTokens) {
        $lexer = $this->getLexer();
-        $lexer->startLexing('<?php ' . $code);
-
-        $tokens = [];
-        while (0 !== $token = $lexer->getNextToken($text)) {
-            $tokens[] = [$token, $text];
-        }
+        $tokens = $lexer->tokenize('<?php ' . $code);
+        // Drop <?php and EOF tokens.
+        $tokens = array_slice($tokens, 1, -1);
+        $tokens = array_map(function(Token $token) {
+            return [$token->id, $token->value];
+        }, $tokens);
        $this->assertSame($expectedTokens, $tokens);
    }

@ -91,11 +108,16 @@ class EmulativeTest extends LexerTest
        $stringifiedToken = '"' . addcslashes($code, '"\\') . '"';

        $lexer = $this->getLexer();
-        $lexer->startLexing('<?php ' . $stringifiedToken);
-
-        $this->assertSame(Tokens::T_CONSTANT_ENCAPSED_STRING, $lexer->getNextToken($text));
-        $this->assertSame($stringifiedToken, $text);
-        $this->assertSame(0, $lexer->getNextToken());
+        $tokens = $lexer->tokenize('<?php ' . $stringifiedToken);
+        $this->assertEquals([
+            new Token(Tokens::T_OPEN_TAG, '<?php ', 1, 0),
+            new Token(Tokens::T_CONSTANT_ENCAPSED_STRING, $stringifiedToken, 1, 6),
+            new Token(
+                0, "\0",
+                1 + substr_count($stringifiedToken, "\n"),
+                6 + strlen($stringifiedToken)
+            ),
+        ], $tokens);
    }

    /**
@ -104,7 +126,7 @@ class EmulativeTest extends LexerTest
    public function testErrorAfterEmulation($code) {
        $errorHandler = new ErrorHandler\Collecting;
        $lexer = $this->getLexer();
-        $lexer->startLexing('<?php ' . $code . "\0", $errorHandler);
+        $lexer->tokenize('<?php ' . $code . "\0", $errorHandler);

        $errors = $errorHandler->getErrors();
        $this->assertCount(1, $errors);
@ -157,12 +179,14 @@ class EmulativeTest extends LexerTest
                [Tokens::T_START_HEREDOC, "<<<'NOWDOC'\n"],
                [Tokens::T_END_HEREDOC, 'NOWDOC'],
                [ord(';'), ';'],
+                [Tokens::T_WHITESPACE, "\n"],
            ]],
            ["<<<'NOWDOC'\nFoobar\nNOWDOC;\n", [
                [Tokens::T_START_HEREDOC, "<<<'NOWDOC'\n"],
                [Tokens::T_ENCAPSED_AND_WHITESPACE, "Foobar\n"],
                [Tokens::T_END_HEREDOC, 'NOWDOC'],
                [ord(';'), ';'],
+                [Tokens::T_WHITESPACE, "\n"],
            ]],

            // PHP 7.3: Flexible heredoc/nowdoc
@ -194,12 +218,15 @@ class EmulativeTest extends LexerTest
                [Tokens::T_START_HEREDOC, "<<<LABEL\n"],
                [Tokens::T_ENCAPSED_AND_WHITESPACE, "LABELNOPE\n"],
                [Tokens::T_END_HEREDOC, "LABEL"],
+                [Tokens::T_WHITESPACE, "\n"],
            ]],
            // Interpretation changed
            ["<<<LABEL\n    LABEL\nLABEL\n", [
                [Tokens::T_START_HEREDOC, "<<<LABEL\n"],
                [Tokens::T_END_HEREDOC, "    LABEL"],
+                [Tokens::T_WHITESPACE, "\n"],
                [Tokens::T_STRING, "LABEL"],
+                [Tokens::T_WHITESPACE, "\n"],
            ]],

            // PHP 7.4: Null coalesce equal
--- a/test/PhpParser/LexerTest.php
+++ b/test/PhpParser/LexerTest.php
@ -11,19 +11,30 @@ class LexerTest extends \PHPUnit\Framework\TestCase
        return new Lexer($options);
    }

+    public function testTokenize() {
+        $code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
+        $expectedTokens = [
+            new Token(Tokens::T_OPEN_TAG, '<?php ', 1, 0),
+            new Token(Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"', 1, 6),
+            new Token(\ord(';'), ';', 1, 9),
+            new Token(Tokens::T_WHITESPACE, "\n", 1, 10),
+            new Token(Tokens::T_COMMENT, '// foo' . "\n", 2, 11),
+            new Token(Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"', 3, 18),
+            new Token(\ord(';'), ';', 3, 21),
+            new Token(0, "\0", 3, 22),
+        ];
+
+        $lexer = $this->getLexer();
+        $this->assertEquals($expectedTokens, $lexer->tokenize($code));
+    }
+
    /**
     * @dataProvider provideTestError
     */
    public function testError($code, $messages) {
-        if (defined('HHVM_VERSION')) {
-            $this->markTestSkipped('HHVM does not throw warnings from token_get_all()');
-        }
-
        $errorHandler = new ErrorHandler\Collecting();
-        $lexer = $this->getLexer(['usedAttributes' => [
-            'comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'
-        ]]);
-        $lexer->startLexing($code, $errorHandler);
+        $lexer = $this->getLexer();
+        $lexer->tokenize($code, $errorHandler);
        $errors = $errorHandler->getErrors();

        $this->assertCount(count($messages), $errors);
@ -46,218 +57,4 @@ class LexerTest extends \PHPUnit\Framework\TestCase
            ]],
        ];
    }
-
-    /**
-     * @dataProvider provideTestLex
-     */
-    public function testLex($code, $options, $tokens) {
-        $lexer = $this->getLexer($options);
-        $lexer->startLexing($code);
-        while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
-            $token = array_shift($tokens);
-
-            $this->assertSame($token[0], $id);
-            $this->assertSame($token[1], $value);
-            $this->assertEquals($token[2], $startAttributes);
-            $this->assertEquals($token[3], $endAttributes);
-        }
-    }
-
-    public function provideTestLex() {
-        return [
-            // tests conversion of closing PHP tag and drop of whitespace and opening tags
-            [
-                '<?php tokens ?>plaintext',
-                [],
-                [
-                    [
-                        Tokens::T_STRING, 'tokens',
-                        ['startLine' => 1], ['endLine' => 1]
-                    ],
-                    [
-                        ord(';'), '?>',
-                        ['startLine' => 1], ['endLine' => 1]
-                    ],
-                    [
-                        Tokens::T_INLINE_HTML, 'plaintext',
-                        ['startLine' => 1, 'hasLeadingNewline' => false],
-                        ['endLine' => 1]
-                    ],
-                ]
-            ],
-            // tests line numbers
-            [
-                '<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $',
-                [],
-                [
-                    [
-                        ord('$'), '$',
-                        ['startLine' => 2], ['endLine' => 2]
-                    ],
-                    [
-                        Tokens::T_STRING, 'token',
-                        ['startLine' => 2], ['endLine' => 2]
-                    ],
-                    [
-                        ord('$'), '$',
-                        [
-                            'startLine' => 3,
-                            'comments' => [
-                                new Comment\Doc('/** doc' . "\n" . 'comment */', 2, 14, 5),
-                            ]
-                        ],
-                        ['endLine' => 3]
-                    ],
-                ]
-            ],
-            // tests comment extraction
-            [
-                '<?php /* comment */ // comment' . "\n" . '/** docComment 1 *//** docComment 2 */ token',
-                [],
-                [
-                    [
-                        Tokens::T_STRING, 'token',
-                        [
-                            'startLine' => 2,
-                            'comments' => [
-                                new Comment('/* comment */', 1, 6, 1),
-                                new Comment('// comment' . "\n", 1, 20, 3),
-                                new Comment\Doc('/** docComment 1 */', 2, 31, 4),
-                                new Comment\Doc('/** docComment 2 */', 2, 50, 5),
-                            ],
-                        ],
-                        ['endLine' => 2]
-                    ],
-                ]
-            ],
-            // tests differing start and end line
-            [
-                '<?php "foo' . "\n" . 'bar"',
-                [],
-                [
-                    [
-                        Tokens::T_CONSTANT_ENCAPSED_STRING, '"foo' . "\n" . 'bar"',
-                        ['startLine' => 1], ['endLine' => 2]
-                    ],
-                ]
-            ],
-            // tests exact file offsets
-            [
-                '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
-                ['usedAttributes' => ['startFilePos', 'endFilePos']],
-                [
-                    [
-                        Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
-                        ['startFilePos' => 6], ['endFilePos' => 8]
-                    ],
-                    [
-                        ord(';'), ';',
-                        ['startFilePos' => 9], ['endFilePos' => 9]
-                    ],
-                    [
-                        Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
-                        ['startFilePos' => 18], ['endFilePos' => 20]
-                    ],
-                    [
-                        ord(';'), ';',
-                        ['startFilePos' => 21], ['endFilePos' => 21]
-                    ],
-                ]
-            ],
-            // tests token offsets
-            [
-                '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
-                ['usedAttributes' => ['startTokenPos', 'endTokenPos']],
-                [
-                    [
-                        Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
-                        ['startTokenPos' => 1], ['endTokenPos' => 1]
-                    ],
-                    [
-                        ord(';'), ';',
-                        ['startTokenPos' => 2], ['endTokenPos' => 2]
-                    ],
-                    [
-                        Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
-                        ['startTokenPos' => 5], ['endTokenPos' => 5]
-                    ],
-                    [
-                        ord(';'), ';',
-                        ['startTokenPos' => 6], ['endTokenPos' => 6]
-                    ],
-                ]
-            ],
-            // tests all attributes being disabled
-            [
-                '<?php /* foo */ $bar;',
-                ['usedAttributes' => []],
-                [
-                    [
-                        Tokens::T_VARIABLE, '$bar',
-                        [], []
-                    ],
-                    [
-                        ord(';'), ';',
-                        [], []
-                    ]
-                ]
-            ],
-            // tests no tokens
-            [
-                '',
-                [],
-                []
-            ],
-        ];
-    }
-
-    /**
-     * @dataProvider provideTestHaltCompiler
-     */
-    public function testHandleHaltCompiler($code, $remaining) {
-        $lexer = $this->getLexer();
-        $lexer->startLexing($code);
-
-        while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
-
-        $this->assertSame($remaining, $lexer->handleHaltCompiler());
-        $this->assertSame(0, $lexer->getNextToken());
-    }
-
-    public function provideTestHaltCompiler() {
-        return [
-            ['<?php ... __halt_compiler();Remaining Text', 'Remaining Text'],
-            ['<?php ... __halt_compiler ( ) ;Remaining Text', 'Remaining Text'],
-            ['<?php ... __halt_compiler() ?>Remaining Text', 'Remaining Text'],
-            //array('<?php ... __halt_compiler();' . "\0", "\0"),
-            //array('<?php ... __halt_compiler /* */ ( ) ;Remaining Text', 'Remaining Text'),
-        ];
-    }
-
-    public function testHandleHaltCompilerError() {
-        $this->expectException(Error::class);
-        $this->expectExceptionMessage('__HALT_COMPILER must be followed by "();"');
-        $lexer = $this->getLexer();
-        $lexer->startLexing('<?php ... __halt_compiler invalid ();');
-
-        while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
-        $lexer->handleHaltCompiler();
-    }
-
-    public function testGetTokens() {
-        $code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
-        $expectedTokens = [
-            [T_OPEN_TAG, '<?php ', 1],
-            [T_CONSTANT_ENCAPSED_STRING, '"a"', 1],
-            ';',
-            [T_WHITESPACE, "\n", 1],
-            [T_COMMENT, '// foo' . "\n", 2],
-            [T_CONSTANT_ENCAPSED_STRING, '"b"', 3],
-            ';',
-        ];
-
-        $lexer = $this->getLexer();
-        $lexer->startLexing($code);
-        $this->assertSame($expectedTokens, $lexer->getTokens());
-    }
 }
Author	SHA1	Message	Date
Nikita Popov	1626c19d98	Update lexer implementations and tests	2019-06-30 23:07:22 +02:00
Nikita Popov	f52785a2b8	Start on lexer refactoring	2019-06-30 19:31:22 +02:00
Nikita Popov	a21a614737	WIP	2019-06-30 17:21:55 +02:00
Nikita Popov	6f74784e16	Switch to a normalized token representation Each token is now represented by a Token object.	2019-06-30 14:14:24 +02:00