mirror_php/PHP-Parser
1
0
Fork 0
mirror of https://github.com/nikic/PHP-Parser.git synced 2025-01-17 07:08:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Nikita Popov 2019-06-30 17:21:55 +02:00
parent 6f74784e16
commit a21a614737
4 changed files with 51 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
composer.json
View File

@ -14,6 +14,7 @@
],
"require": {
"php": ">=7.0",
"ext-json": "*",
"ext-tokenizer": "*"
},
"require-dev": {

8
lib/PhpParser/FileContext.php Normal file
View File

@ -0,0 +1,8 @@
<?php declare(strict_types=1);
namespace PhpParser;
class FileContext {
/** @var Token[] */
public $tokens;
}

53
lib/PhpParser/Lexer.php
View File

@ -227,23 +227,12 @@ class Lexer
*
* @return int Token id
*/
public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
public function getNextToken(&$value = null, &$startAttributes = null) : int {
$startAttributes = [];
$endAttributes = [];
while (1) {
$token = $this->tokens[++$this->pos];
if ($this->attributeStartLineUsed) {
$startAttributes['startLine'] = $token->line;
}
if ($this->attributeStartTokenPosUsed) {
$startAttributes['startTokenPos'] = $this->pos;
}
if ($this->attributeStartFilePosUsed) {
$startAttributes['startFilePos'] = $token->filePos;
}
$phpId = $token->id;
$value = $token->value;
if (!isset($this->dropTokens[$phpId])) {
@ -254,16 +243,6 @@ class Lexer
$startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
}
if ($this->attributeEndLineUsed) {
$endAttributes['endLine'] = $token->line + substr_count($value, "\n");
}
if ($this->attributeEndTokenPosUsed) {
$endAttributes['endTokenPos'] = $this->pos;
}
if ($this->attributeEndFilePosUsed) {
$endAttributes['endFilePos'] = $token->filePos + \strlen($value) - 1;
}
return $id;
}
@ -281,19 +260,18 @@ class Lexer
}
/**
* Returns the token array for current code.
* Returns the token array for the current code.
*
* The token array is in the same format as provided by the
* token_get_all() function and does not discard tokens (i.e.
* whitespace and comments are included). The token position
* attributes are against this token array.
*
* @return array Array of tokens in token_get_all() format
* @return Token[]
*/
public function getTokens() : array {
return $this->tokens;
}
public function getTokenMap(): array {
return $this->tokenMap;
}
/**
* Handles __halt_compiler() by returning the text after it.
*
@ -324,6 +302,8 @@ class Lexer
* The token map maps the PHP internal token identifiers
* to the identifiers used by the Parser. Additionally it
* maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
* Whitespace and comment tokens are mapped to null, which indicates
* that they should be dropped.
*
* @return array The token map
*/
@ -346,23 +326,16 @@ class Lexer
// T_CLOSE_TAG is equivalent to ';'
$tokenMap[$i] = ord(';');
} elseif ('UNKNOWN' !== $name = token_name($i)) {
if ('T_HASHBANG' === $name) {
// HHVM uses a special token for #! hashbang lines
$tokenMap[$i] = Tokens::T_INLINE_HTML;
} elseif (defined($name = Tokens::class . '::' . $name)) {
if (defined($name = Tokens::class . '::' . $name)) {
// Other tokens can be mapped directly
$tokenMap[$i] = constant($name);
}
}
}
// HHVM uses a special token for numbers that overflow to double
if (defined('T_ONUMBER')) {
$tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
}
// HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
if (defined('T_COMPILER_HALT_OFFSET')) {
$tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
$dropTokens = [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, self::T_BAD_CHARACTER];
foreach ($dropTokens as $dropToken) {
$tokenMap[$dropToken] = null;
}
return $tokenMap;

48
lib/PhpParser/NodeAbstract.php
View File

@ -4,8 +4,14 @@ namespace PhpParser;
abstract class NodeAbstract implements Node, \JsonSerializable
{
// TODO: Kill.
protected $attributes;
/** @var FileContext|null */
protected $context = null;
protected $startTokenPos = -1;
protected $endTokenPos = -1;
/**
* Creates a Node.
*
@ -15,35 +21,42 @@ abstract class NodeAbstract implements Node, \JsonSerializable
$this->attributes = $attributes;
}
public function setTokenContext(FileContext $context, int $firstToken, int $lastToken) {
$this->context = $context;
$this->startTokenPos = $firstToken;
$this->endTokenPos = $lastToken;
}
/**
* Gets line the node started in (alias of getStartLine).
*
* @return int Start line (or -1 if not available)
*/
public function getLine() : int {
return $this->attributes['startLine'] ?? -1;
return $this->context->tokens[$this->startTokenPos]->line ?? -1;
}
/**
* Gets line the node started in.
*
* Requires the 'startLine' attribute to be enabled in the lexer (enabled by default).
*
* @return int Start line (or -1 if not available)
*/
public function getStartLine() : int {
return $this->attributes['startLine'] ?? -1;
return $this->context->tokens[$this->startTokenPos]->line ?? -1;
}
/**
* Gets the line the node ended in.
*
* Requires the 'endLine' attribute to be enabled in the lexer (enabled by default).
*
* @return int End line (or -1 if not available)
*/
public function getEndLine() : int {
return $this->attributes['endLine'] ?? -1;
if (!isset($this->context->tokens[$this->endTokenPos])) {
return -1;
}
$token = $this->context->tokens[$this->endTokenPos];
return $token->line + \substr_count($token, "\n");
}
/**
@ -51,12 +64,10 @@ abstract class NodeAbstract implements Node, \JsonSerializable
*
* The offset is an index into the array returned by Lexer::getTokens().
*
* Requires the 'startTokenPos' attribute to be enabled in the lexer (DISABLED by default).
*
* @return int Token start position (or -1 if not available)
*/
public function getStartTokenPos() : int {
return $this->attributes['startTokenPos'] ?? -1;
return $this->startTokenPos;
}
/**
@ -64,34 +75,33 @@ abstract class NodeAbstract implements Node, \JsonSerializable
*
* The offset is an index into the array returned by Lexer::getTokens().
*
* Requires the 'endTokenPos' attribute to be enabled in the lexer (DISABLED by default).
*
* @return int Token end position (or -1 if not available)
*/
public function getEndTokenPos() : int {
return $this->attributes['endTokenPos'] ?? -1;
return $this->endTokenPos;
}
/**
* Gets the file offset of the first character that is part of this node.
*
* Requires the 'startFilePos' attribute to be enabled in the lexer (DISABLED by default).
*
* @return int File start position (or -1 if not available)
*/
public function getStartFilePos() : int {
return $this->attributes['startFilePos'] ?? -1;
return $this->context->tokens[$this->startTokenPos]->filePos ?? -1;
}
/**
* Gets the file offset of the last character that is part of this node.
*
* Requires the 'endFilePos' attribute to be enabled in the lexer (DISABLED by default).
*
* @return int File end position (or -1 if not available)
*/
public function getEndFilePos() : int {
return $this->attributes['endFilePos'] ?? -1;
if (!isset($this->context->tokens[$this->endTokenPos])) {
return -1;
}
$token = $this->context->tokens[$this->endTokenPos];
return $token->filePos + \strlen($token->value) - 1;
}
/**