mirror_php/PHP-Parser
1
0
Fork 0
mirror of https://github.com/nikic/PHP-Parser.git synced 2025-01-16 22:58:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

Move attribute handling into parser

Browse Source 
The Lexer now only provides the tokens to the parser, while the
parser is responsible for determining which attributes are placed
on notes. This only needs to be done when the attributes are
actually needed, rather than for all tokens.

This removes the usedAttributes lexer option (and lexer options
entirely). The attributes are now enabled unconditionally. They
have less overhead now, and the need to explicitly enable them for
some use cases (e.g. formatting-preserving printing) doesn't seem
like a good tradeoff anymore.

There are some additional changes to the Lexer interface that
should be done after this, and the docs / upgrading guide haven't
been adjusted yet.
This commit is contained in:
Nikita Popov 2023-07-29 15:10:11 +02:00
parent b20267c5ad
commit 4b497045e0
18 changed files with 958 additions and 1130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bin/php-parse
View File

@ -26,10 +26,7 @@ if (empty($files)) {
showHelp("Must specify at least one file.");
}
$lexerOptions = ['usedAttributes' => [
'startLine', 'endLine', 'startFilePos', 'endFilePos', 'comments'
]];
$parser = (new PhpParser\ParserFactory())->createForVersion($attributes['version'], $lexerOptions);
$parser = (new PhpParser\ParserFactory())->createForVersion($attributes['version']);
$dumper = new PhpParser\NodeDumper([
'dumpComments' => true,
'dumpPositions' => $attributes['with-positions'],

19
grammar/php.y
View File

@ -134,7 +134,7 @@ top_statement_list_ex:
top_statement_list:
top_statement_list_ex
{ makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
{ makeZeroLengthNop($nop);
if ($nop !== null) { $1[] = $nop; } $$ = $1; }
;
@ -237,7 +237,7 @@ top_statement:
| function_declaration_statement
| class_declaration_statement
| T_HALT_COMPILER '(' ')' ';'
{ $$ = Stmt\HaltCompiler[$this->lexer->handleHaltCompiler()]; }
{ $$ = Stmt\HaltCompiler[$this->handleHaltCompiler()]; }
| T_NAMESPACE namespace_declaration_name semi
{ $$ = Stmt\Namespace_[$2, null];
$$->setAttribute('kind', Stmt\Namespace_::KIND_SEMICOLON);
@ -353,7 +353,7 @@ inner_statement_list_ex:
inner_statement_list:
inner_statement_list_ex
{ makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
{ makeZeroLengthNop($nop);
if ($nop !== null) { $1[] = $nop; } $$ = $1; }
;
@ -371,7 +371,7 @@ non_empty_statement:
if ($2) {
$$ = $2; prependLeadingComments($$);
} else {
makeNop($$, $this->startAttributeStack[#1], $this->endAttributes);
makeNop($$);
if (null === $$) { $$ = array(); }
}
}
@ -390,7 +390,10 @@ non_empty_statement:
| T_GLOBAL global_var_list semi { $$ = Stmt\Global_[$2]; }
| T_STATIC static_var_list semi { $$ = Stmt\Static_[$2]; }
| T_ECHO expr_list_forbid_comma semi { $$ = Stmt\Echo_[$2]; }
| T_INLINE_HTML { $$ = Stmt\InlineHTML[$1]; }
| T_INLINE_HTML {
$$ = Stmt\InlineHTML[$1];
$$->setAttribute('hasLeadingNewline', $this->inlineHtmlHasLeadingNewline(#1));
}
| expr semi {
$e = $1;
if ($e instanceof Expr\Throw_) {
@ -419,7 +422,7 @@ non_empty_statement:
statement:
non_empty_statement
| ';'
{ makeNop($$, $this->startAttributeStack[#1], $this->endAttributes);
{ makeNop($$);
if ($$ === null) $$ = array(); /* means: no statement */ }
;
@ -834,7 +837,7 @@ class_statement_list_ex:
class_statement_list:
class_statement_list_ex
{ makeZeroLengthNop($nop, $this->lookaheadStartAttributes);
{ makeZeroLengthNop($nop);
if ($nop !== null) { $1[] = $nop; } $$ = $1; }
;
@ -1337,7 +1340,7 @@ array_pair:
| /* empty */
{ /* Create an Error node now to remember the position. We'll later either report an error,
or convert this into a null element, depending on whether this is a creation or destructuring context. */
$attrs = $this->createEmptyElemAttributes($this->lookaheadStartAttributes);
$attrs = $this->createEmptyElemAttributes($this->tokenPos);
$$ = new Node\ArrayItem(new Expr\Error($attrs), null, false, $attrs); }
;

26
grammar/phpyLang.php
View File

@ -65,13 +65,13 @@ function resolveMacros($code) {
if ('attributes' === $name) {
assertArgs(0, $args, $name);
return '$this->startAttributeStack[#1] + $this->endAttributes';
return '$this->getAttributes($this->tokenStartStack[#1], $this->tokenEndStack[$stackPos])';
}
if ('stackAttributes' === $name) {
assertArgs(1, $args, $name);
return '$this->startAttributeStack[' . $args[0] . ']'
. ' + $this->endAttributeStack[' . $args[0] . ']';
return '$this->getAttributes($this->tokenStartStack[' . $args[0] . '], '
. ' $this->tokenEndStack[' . $args[0] . '])';
}
if ('init' === $name) {
@ -111,30 +111,24 @@ function resolveMacros($code) {
}
if ('makeNop' === $name) {
assertArgs(3, $args, $name);
assertArgs(1, $args, $name);
return '$startAttributes = ' . $args[1] . ';'
. ' if (isset($startAttributes[\'comments\']))'
. ' { ' . $args[0] . ' = new Stmt\Nop($startAttributes + ' . $args[2] . '); }'
. ' else { ' . $args[0] . ' = null; }';
return $args[0] . ' = $this->maybeCreateNop($this->tokenStartStack[#1], $this->tokenEndStack[$stackPos])';
}
if ('makeZeroLengthNop' == $name) {
assertArgs(2, $args, $name);
assertArgs(1, $args, $name);
return '$startAttributes = ' . $args[1] . ';'
. ' if (isset($startAttributes[\'comments\']))'
. ' { ' . $args[0] . ' = new Stmt\Nop($this->createCommentNopAttributes($startAttributes[\'comments\'])); }'
. ' else { ' . $args[0] . ' = null; }';
return $args[0] . ' = $this->maybeCreateZeroLengthNop($this->tokenPos);';
}
if ('prependLeadingComments' === $name) {
assertArgs(1, $args, $name);
return '$attrs = $this->startAttributeStack[#1]; $stmts = ' . $args[0] . '; '
. 'if (!empty($attrs[\'comments\'])) {'
return '$comments = $this->getCommentsBeforeToken($this->tokenStartStack[#1]); $stmts = ' . $args[0] . '; '
. 'if (!empty($comments)) {'
. '$stmts[0]->setAttribute(\'comments\', '
. 'array_merge($attrs[\'comments\'], $stmts[0]->getAttribute(\'comments\', []))); }';
. 'array_merge($comments, $stmts[0]->getAttribute(\'comments\', []))); }';
}
return $matches[0];

151
lib/PhpParser/Lexer.php
View File

@ -5,60 +5,8 @@ namespace PhpParser;
require __DIR__ . '/compatibility_tokens.php';
class Lexer {
/** @var string Code being tokenized */
protected $code;
/** @var list<Token> List of tokens */
protected $tokens;
/** @var int Current position in the token array */
protected $pos;
/** @var bool Whether the preceding closing PHP tag has a trailing newline */
protected $prevCloseTagHasNewline;
/** @var array<int, int> Map of tokens that should be dropped (like T_WHITESPACE) */
protected $dropTokens;
/** @var bool Whether to use the startLine attribute */
private $attributeStartLineUsed;
/** @var bool Whether to use the endLine attribute */
private $attributeEndLineUsed;
/** @var bool Whether to use the startTokenPos attribute */
private $attributeStartTokenPosUsed;
/** @var bool Whether to use the endTokenPos attribute */
private $attributeEndTokenPosUsed;
/** @var bool Whether to use the startFilePos attribute */
private $attributeStartFilePosUsed;
/** @var bool Whether to use the endFilePos attribute */
private $attributeEndFilePosUsed;
/** @var bool Whether to use the comments attribute */
private $attributeCommentsUsed;
/**
* Creates a Lexer.
*
* @param array{usedAttributes?: string[]} $options Options array. Currently only the
* 'usedAttributes' option is supported, which is an array of attributes to add to the
* AST nodes. Possible attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
* 'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the first three.
* For more info see getNextToken() docs.
*/
public function __construct(array $options = []) {
// map of tokens to drop while lexing (the map is only used for isset lookup,
// that's why the value is simply set to 1; the value is never actually used.)
$this->dropTokens = array_fill_keys(
[\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
);
$defaultAttributes = ['comments', 'startLine', 'endLine'];
$usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
// Create individual boolean properties to make these checks faster.
$this->attributeStartLineUsed = isset($usedAttributes['startLine']);
$this->attributeEndLineUsed = isset($usedAttributes['endLine']);
$this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
$this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
$this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
$this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
$this->attributeCommentsUsed = isset($usedAttributes['comments']);
}
/**
* Initializes the lexer for lexing the provided source code.
@ -75,13 +23,6 @@ class Lexer {
$errorHandler = new ErrorHandler\Throwing();
}
$this->code = $code; // keep the code around for __halt_compiler() handling
$this->pos = -1;
// If inline HTML occurs without preceding code, treat it as if it had a leading newline.
// This ensures proper composability, because having a newline is the "safe" assumption.
$this->prevCloseTagHasNewline = true;
$scream = ini_set('xdebug.scream', '0');
$this->tokens = @Token::tokenize($code);
@ -165,84 +106,6 @@ class Lexer {
$this->tokens[] = new Token(0, "\0", $lastToken->getEndLine(), $lastToken->getEndPos());
}
/**
* Fetches the next token.
*
* The available attributes are determined by the 'usedAttributes' option, which can
* be specified in the constructor. The following attributes are supported:
*
* * 'comments' => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
* representing all comments that occurred between the previous
* non-discarded token and the current one.
* * 'startLine' => Line in which the node starts.
* * 'endLine' => Line in which the node ends.
* * 'startTokenPos' => Offset into the token array of the first token in the node.
* * 'endTokenPos' => Offset into the token array of the last token in the node.
* * 'startFilePos' => Offset into the code string of the first character that is part of the node.
* * 'endFilePos' => Offset into the code string of the last character that is part of the node.
*
* @param mixed $value Variable to store token content in
* @param mixed $startAttributes Variable to store start attributes in
* @param mixed $endAttributes Variable to store end attributes in
*
* @return int Token id
*/
public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null): int {
$startAttributes = [];
$endAttributes = [];
while (1) {
$token = $this->tokens[++$this->pos];
$id = $token->id;
if (isset($this->dropTokens[$id])) {
if (\T_COMMENT === $id || \T_DOC_COMMENT === $id) {
if ($this->attributeCommentsUsed) {
$comment = \T_DOC_COMMENT === $id
? new Comment\Doc($token->text, $token->line, $token->pos, $this->pos,
$token->getEndLine(), $token->getEndPos() - 1, $this->pos)
: new Comment($token->text, $token->line, $token->pos, $this->pos,
$token->getEndLine(), $token->getEndPos() - 1, $this->pos);
$startAttributes['comments'][] = $comment;
}
}
continue;
}
if ($this->attributeStartLineUsed) {
$startAttributes['startLine'] = $token->line;
}
if ($this->attributeStartTokenPosUsed) {
$startAttributes['startTokenPos'] = $this->pos;
}
if ($this->attributeStartFilePosUsed) {
$startAttributes['startFilePos'] = $token->pos;
}
$value = $token->text;
if (\T_CLOSE_TAG === $token->id) {
$this->prevCloseTagHasNewline = false !== strpos($value, "\n")
|| false !== strpos($value, "\r");
} elseif (\T_INLINE_HTML === $token->id) {
$startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
}
// Fetch the end line/pos from the next token (if available) instead of recomputing it.
$nextToken = $this->tokens[$this->pos + 1] ?? null;
if ($this->attributeEndLineUsed) {
$endAttributes['endLine'] = $nextToken ? $nextToken->line : $token->getEndLine();
}
if ($this->attributeEndTokenPosUsed) {
$endAttributes['endTokenPos'] = $this->pos;
}
if ($this->attributeEndFilePosUsed) {
$endAttributes['endFilePos'] = ($nextToken ? $nextToken->pos : $token->getEndPos()) - 1;
}
return $id;
}
}
/**
* Returns the token array for current code.
*
@ -259,18 +122,4 @@ class Lexer {
public function getTokens(): array {
return $this->tokens;
}
/**
* Handles __halt_compiler() by returning the text after it.
*
* @return string Remaining text
*/
public function handleHaltCompiler(): string {
// Prevent the lexer from returning any further tokens.
$nextToken = $this->tokens[$this->pos + 1];
$this->pos = \count($this->tokens) - 2;
// Return text after __halt_compiler.
return $nextToken->id === \T_INLINE_HTML ? $nextToken->text : '';
}
}

15
lib/PhpParser/Lexer/Emulative.php
View File

@ -33,20 +33,11 @@ class Emulative extends Lexer {
private $hostPhpVersion;
/**
* @param array{usedAttributes?: string[], phpVersion?: PhpVersion|string} $options Lexer options.
* In addition to the usual options, accepts a 'phpVersion' (PhpVersion object or string)
* that specifies the version to emulate. Defaults to newest supported.
* @param PhpVersion|null $phpVersion PHP version to emulate. Defaults to newest supported.
*/
public function __construct(array $options = []) {
$version = $options['phpVersion'] ?? PhpVersion::getNewestSupported();
if (!$version instanceof PhpVersion) {
$version = PhpVersion::fromString($version);
}
$this->targetPhpVersion = $version;
public function __construct(?PhpVersion $phpVersion = null) {
$this->targetPhpVersion = $phpVersion ?? PhpVersion::getNewestSupported();
$this->hostPhpVersion = PhpVersion::getHostVersion();
unset($options['phpVersion']);
parent::__construct($options);
$emulators = [
new FlexibleDocStringEmulator(),

563
lib/PhpParser/Parser/Php7.php
View File

File diff suppressed because it is too large Load Diff

563
lib/PhpParser/Parser/Php8.php
View File

File diff suppressed because it is too large Load Diff

227
lib/PhpParser/ParserAbstract.php
View File

@ -66,6 +66,8 @@ abstract class ParserAbstract implements Parser {
/** @var int[] Map of PHP token IDs to internal symbols */
protected $phpTokenToSymbol;
/** @var array<int, bool> Map of PHP token IDs to drop */
protected $dropTokens;
/** @var int[] Map of external symbols (static::T_*) to internal symbols */
protected $tokenToSymbol;
/** @var string[] Map of symbols to their names */
@ -113,14 +115,10 @@ abstract class ParserAbstract implements Parser {
protected $semValue;
/** @var mixed[] Semantic value stack (contains values of tokens and semantic action results) */
protected $semStack;
/** @var array<string, mixed>[] Start attribute stack */
protected $startAttributeStack;
/** @var array<string, mixed>[] End attribute stack */
protected $endAttributeStack;
/** @var array<string, mixed> End attributes of last *shifted* token */
protected $endAttributes;
/** @var array<string, mixed> Start attributes of last *read* token */
protected $lookaheadStartAttributes;
/** @var int[] Token start position stack */
protected $tokenStartStack;
/** @var int[] Token end position stack */
protected $tokenEndStack;
/** @var ErrorHandler Error handler */
protected $errorHandler;
@ -130,6 +128,11 @@ abstract class ParserAbstract implements Parser {
/** @var \SplObjectStorage<Array_, null>|null Array nodes created during parsing, for postprocessing of empty elements. */
protected $createdArrays;
/** @var Token[] Tokens for the current parse */
protected $tokens;
/** @var int Current position in token array */
protected $tokenPos;
/**
* Initialize $reduceCallbacks map.
*/
@ -154,6 +157,9 @@ abstract class ParserAbstract implements Parser {
$this->initReduceCallbacks();
$this->phpTokenToSymbol = $this->createTokenMap();
$this->dropTokens = array_fill_keys(
[\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
);
}
/**
@ -174,6 +180,7 @@ abstract class ParserAbstract implements Parser {
$this->createdArrays = new \SplObjectStorage();
$this->lexer->startLexing($code, $this->errorHandler);
$this->tokens = $this->lexer->getTokens();
$result = $this->doParse();
// Report errors for any empty elements used inside arrays. This is delayed until after the main parse,
@ -190,8 +197,9 @@ abstract class ParserAbstract implements Parser {
// Clear out some of the interior state, so we don't hold onto unnecessary
// memory between uses of the parser
$this->startAttributeStack = [];
$this->endAttributeStack = [];
$this->tokens = [];
$this->tokenStartStack = [];
$this->tokenEndStack = [];
$this->semStack = [];
$this->semValue = null;
$this->createdArrays = null;
@ -207,17 +215,12 @@ abstract class ParserAbstract implements Parser {
protected function doParse(): ?array {
// We start off with no lookahead-token
$symbol = self::SYMBOL_NONE;
// The attributes for a node are taken from the first and last token of the node.
// From the first token only the startAttributes are taken and from the last only
// the endAttributes. Both are merged using the array union operator (+).
$startAttributes = [];
$endAttributes = [];
$this->endAttributes = $endAttributes;
$tokenValue = null;
$this->tokenPos = -1;
// Keep stack of start and end attributes
$this->startAttributeStack = [];
$this->endAttributeStack = [$endAttributes];
$this->tokenStartStack = [];
$this->tokenEndStack = [0];
// Start off in the initial state and keep a stack of previous states
$state = 0;
@ -238,13 +241,13 @@ abstract class ParserAbstract implements Parser {
$rule = $this->actionDefault[$state];
} else {
if ($symbol === self::SYMBOL_NONE) {
// Fetch the next token id from the lexer and fetch additional info by-ref.
// The end attributes are fetched into a temporary variable and only set once the token is really
// shifted (not during read). Otherwise you would sometimes get off-by-one errors, when a rule is
// reduced after a token was read but not yet shifted.
$tokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $endAttributes);
do {
$token = $this->tokens[++$this->tokenPos];
$tokenId = $token->id;
} while (isset($this->dropTokens[$tokenId]));
// Map the lexer token id to the internally used symbols.
$tokenValue = $token->text;
if (!isset($this->phpTokenToSymbol[$tokenId])) {
throw new \RangeException(sprintf(
'The lexer returned an invalid token (id=%d, value=%s)',
@ -253,9 +256,6 @@ abstract class ParserAbstract implements Parser {
}
$symbol = $this->phpTokenToSymbol[$tokenId];
// Allow productions to access the start attributes of the lookahead token.
$this->lookaheadStartAttributes = $startAttributes;
//$this->traceRead($symbol);
}
@ -279,9 +279,8 @@ abstract class ParserAbstract implements Parser {
++$stackPos;
$stateStack[$stackPos] = $state = $action;
$this->semStack[$stackPos] = $tokenValue;
$this->startAttributeStack[$stackPos] = $startAttributes;
$this->endAttributeStack[$stackPos] = $endAttributes;
$this->endAttributes = $endAttributes;
$this->tokenStartStack[$stackPos] = $this->tokenPos;
$this->tokenEndStack[$stackPos] = $this->tokenPos;
$symbol = self::SYMBOL_NONE;
if ($this->errorState) {
@ -321,8 +320,8 @@ abstract class ParserAbstract implements Parser {
$this->semValue = $this->semStack[$stackPos - $ruleLength + 1];
}
} catch (Error $e) {
if (-1 === $e->getStartLine() && isset($startAttributes['startLine'])) {
$e->setStartLine($startAttributes['startLine']);
if (-1 === $e->getStartLine()) {
$e->setStartLine($this->tokens[$this->tokenPos]->line);
}
$this->emitError($e);
@ -331,7 +330,7 @@ abstract class ParserAbstract implements Parser {
}
/* Goto - shift nonterminal */
$lastEndAttributes = $this->endAttributeStack[$stackPos];
$lastTokenEnd = $this->tokenEndStack[$stackPos];
$stackPos -= $ruleLength;
$nonTerminal = $this->ruleToNonTerminal[$rule];
$idx = $this->gotoBase[$nonTerminal] + $stateStack[$stackPos];
@ -344,17 +343,17 @@ abstract class ParserAbstract implements Parser {
++$stackPos;
$stateStack[$stackPos] = $state;
$this->semStack[$stackPos] = $this->semValue;
$this->endAttributeStack[$stackPos] = $lastEndAttributes;
$this->tokenEndStack[$stackPos] = $lastTokenEnd;
if ($ruleLength === 0) {
// Empty productions use the start attributes of the lookahead token.
$this->startAttributeStack[$stackPos] = $this->lookaheadStartAttributes;
$this->tokenStartStack[$stackPos] = $this->tokenPos;
}
} else {
/* error */
switch ($this->errorState) {
case 0:
$msg = $this->getErrorMessage($symbol, $state);
$this->emitError(new Error($msg, $startAttributes + $endAttributes));
$this->emitError(new Error($msg, $this->getAttributesForToken($this->tokenPos)));
// Break missing intentionally
// no break
case 1:
@ -383,9 +382,8 @@ abstract class ParserAbstract implements Parser {
// We treat the error symbol as being empty, so we reset the end attributes
// to the end attributes of the last non-error symbol
$this->startAttributeStack[$stackPos] = $this->lookaheadStartAttributes;
$this->endAttributeStack[$stackPos] = $this->endAttributeStack[$stackPos - 1];
$this->endAttributes = $this->endAttributeStack[$stackPos - 1];
$this->tokenStartStack[$stackPos] = $this->tokenPos;
$this->tokenEndStack[$stackPos] = $this->tokenEndStack[$stackPos - 1];
break;
case 3:
@ -468,6 +466,53 @@ abstract class ParserAbstract implements Parser {
return $expected;
}
/**
* Get attributes for a node with the given start and end token positions.
*
* @param int $tokenStartPos Token position the node starts at
* @param int $tokenEndPos Token position the node ends at
* @return array Attributes
*/
protected function getAttributes(int $tokenStartPos, int $tokenEndPos): array {
$startToken = $this->tokens[$tokenStartPos];
$afterEndToken = $this->tokens[$tokenEndPos + 1];
$attributes = [
'startLine' => $startToken->line,
'startTokenPos' => $tokenStartPos,
'startFilePos' => $startToken->pos,
'endLine' => $afterEndToken->line,
'endTokenPos' => $tokenEndPos,
'endFilePos' => $afterEndToken->pos - 1,
];
$comments = $this->getCommentsBeforeToken($tokenStartPos);
if (!empty($comments)) {
$attributes['comments'] = $comments;
}
return $attributes;
}
protected function getAttributesForToken(int $tokenPos) {
if ($tokenPos < \count($this->tokens) - 1) {
return $this->getAttributes($tokenPos, $tokenPos);
}
// Get attributes for the sentinel token.
$token = $this->tokens[$tokenPos];
$attributes = [
'startLine' => $token->line,
'startTokenPos' => $tokenPos,
'startFilePos' => $token->pos,
'endLine' => $token->line,
'endTokenPos' => $tokenPos,
'endFilePos' => $token->pos,
];
$comments = $this->getCommentsBeforeToken($tokenPos);
if (!empty($comments)) {
$attributes['comments'] = $comments;
}
return $attributes;
}
/*
* Tracing functions used for debugging the parser.
*/
@ -670,12 +715,12 @@ abstract class ParserAbstract implements Parser {
/**
* Get combined start and end attributes at a stack location
*
* @param int $pos Stack location
* @param int $stackPos Stack location
*
* @return array<string, mixed> Combined start and end attributes
*/
protected function getAttributesAt(int $pos): array {
return $this->startAttributeStack[$pos] + $this->endAttributeStack[$pos];
protected function getAttributesAt(int $stackPos): array {
return $this->getAttributes($this->tokenStartStack[$stackPos], $this->tokenEndStack[$stackPos]);
}
protected function getFloatCastKind(string $cast): int {
@ -842,49 +887,93 @@ abstract class ParserAbstract implements Parser {
}
}
protected function createCommentFromToken(Token $token, int $tokenPos): Comment {
assert($token->id === \T_COMMENT || $token->id == \T_DOC_COMMENT);
return \T_DOC_COMMENT === $token->id
? new Comment\Doc($token->text, $token->line, $token->pos, $tokenPos,
$token->getEndLine(), $token->getEndPos() - 1, $tokenPos)
: new Comment($token->text, $token->line, $token->pos, $tokenPos,
$token->getEndLine(), $token->getEndPos() - 1, $tokenPos);
}
protected function getCommentsBeforeToken(int $tokenPos): array {
$comments = [];
while (--$tokenPos >= 0) {
$token = $this->tokens[$tokenPos];
if (!isset($this->dropTokens[$token->id])) {
break;
}
if ($token->id === \T_COMMENT || $token->id === \T_DOC_COMMENT) {
$comments[] = $this->createCommentFromToken($token, $tokenPos);
}
}
return \array_reverse($comments);
}
/**
* Create attributes for a zero-length common-capturing nop.
* Create a zero-length nop to capture preceding comments, if any.
*
* @param Comment[] $comments
* @return array<string, mixed>
*/
protected function createCommentNopAttributes(array $comments): array {
$comment = $comments[count($comments) - 1];
protected function maybeCreateZeroLengthNop(int $tokenPos): ?Nop {
$comments = $this->getCommentsBeforeToken($tokenPos);
if (empty($comments)) {
return null;
}
$comment = $comments[\count($comments) - 1];
$commentEndLine = $comment->getEndLine();
$commentEndFilePos = $comment->getEndFilePos();
$commentEndTokenPos = $comment->getEndTokenPos();
$attributes = [
'startLine' => $commentEndLine,
'endLine' => $commentEndLine,
'startFilePos' => $commentEndFilePos + 1,
'endFilePos' => $commentEndFilePos,
'startTokenPos' => $commentEndTokenPos + 1,
'endTokenPos' => $commentEndTokenPos,
'comments' => $comments,
];
return new Nop($attributes);
}
$attributes = ['comments' => $comments];
if (-1 !== $commentEndLine) {
$attributes['startLine'] = $commentEndLine;
$attributes['endLine'] = $commentEndLine;
protected function maybeCreateNop(int $tokenStartPos, int $tokenEndPos): ?Nop {
$comments = $this->getCommentsBeforeToken($tokenStartPos);
if (empty($comments)) {
return null;
}
if (-1 !== $commentEndFilePos) {
$attributes['startFilePos'] = $commentEndFilePos + 1;
$attributes['endFilePos'] = $commentEndFilePos;
return new Nop($this->getAttributes($tokenStartPos, $tokenEndPos));
}
protected function handleHaltCompiler(): string {
// Prevent the lexer from returning any further tokens.
$nextToken = $this->tokens[$this->tokenPos + 1];
$this->tokenPos = \count($this->tokens) - 2;
// Return text after __halt_compiler.
return $nextToken->id === \T_INLINE_HTML ? $nextToken->text : '';
}
protected function inlineHtmlHasLeadingNewline(int $stackPos): bool {
$tokenPos = $this->tokenStartStack[$stackPos];
$token = $this->tokens[$tokenPos];
assert($token->id == \T_INLINE_HTML);
if ($tokenPos > 0) {
$prevToken = $this->tokens[$tokenPos - 1];
assert($prevToken->id == \T_CLOSE_TAG);
return false !== strpos($prevToken->text, "\n")
|| false !== strpos($prevToken->text, "\r");
}
if (-1 !== $commentEndTokenPos) {
$attributes['startTokenPos'] = $commentEndTokenPos + 1;
$attributes['endTokenPos'] = $commentEndTokenPos;
}
return $attributes;
return true;
}
/**
* @param array<string, mixed> $attrs
* @return array<string, mixed>
*/
protected function createEmptyElemAttributes(array $attrs): array {
if (isset($attrs['startLine'])) {
$attrs['endLine'] = $attrs['startLine'];
}
if (isset($attrs['startFilePos'])) {
$attrs['endFilePos'] = $attrs['startFilePos'];
}
if (isset($attrs['startTokenPos'])) {
$attrs['endTokenPos'] = $attrs['startTokenPos'];
}
return $attrs;
protected function createEmptyElemAttributes(int $tokenPos): array {
return $this->getAttributesForToken($tokenPos);
}
protected function fixupArrayDestructuring(Array_ $node): Expr\List_ {

20
lib/PhpParser/ParserFactory.php
View File

@ -38,14 +38,12 @@ class ParserFactory {
* Create a parser targeting the given version on a best-effort basis. The parser will generally
* accept code for the newest supported version, but will try to accommodate code that becomes
* invalid in newer versions or changes in interpretation.
*
* @param array<string, mixed> $lexerOptions Lexer options
*/
public function createForVersion(PhpVersion $version, array $lexerOptions = []): Parser {
public function createForVersion(PhpVersion $version): Parser {
if ($version->isHostVersion()) {
$lexer = new Lexer($lexerOptions);
$lexer = new Lexer();
} else {
$lexer = new Lexer\Emulative($lexerOptions + ['phpVersion' => $version]);
$lexer = new Lexer\Emulative($version);
}
if ($version->id >= 80000) {
return new Php8($lexer, $version);
@ -57,20 +55,16 @@ class ParserFactory {
* Create a parser targeting the newest version supported by this library. Code for older
* versions will be accepted if there have been no relevant backwards-compatibility breaks in
* PHP.
*
* @param array<string, mixed> $lexerOptions Lexer options
*/
public function createForNewestSupportedVersion(array $lexerOptions = []): Parser {
return $this->createForVersion(PhpVersion::getNewestSupported(), $lexerOptions);
public function createForNewestSupportedVersion(): Parser {
return $this->createForVersion(PhpVersion::getNewestSupported());
}
/**
* Create a parser targeting the host PHP version, that is the PHP version we're currently
* running on. This parser will not use any token emulation.
*
* @param array<string, mixed> $lexerOptions Lexer options
*/
public function createForHostVersion(array $lexerOptions = []): Parser {
return $this->createForVersion(PhpVersion::getHostVersion(), $lexerOptions);
public function createForHostVersion(): Parser {
return $this->createForVersion(PhpVersion::getHostVersion());
}
}

9
test/PhpParser/CodeParsingTest.php
View File

@ -22,14 +22,7 @@ class CodeParsingTest extends CodeTestAbstract {
$factory = new ParserFactory();
$version = $version === null
? PhpVersion::getNewestSupported() : PhpVersion::fromString($version);
return $factory->createForVersion(
$version,
['usedAttributes' => [
'startLine', 'endLine',
'startFilePos', 'endFilePos',
'startTokenPos', 'endTokenPos',
'comments'
]]);
return $factory->createForVersion($version);
}
// Must be public for updateTests.php

93
test/PhpParser/Lexer/EmulativeTest.php
View File

@ -5,34 +5,44 @@ namespace PhpParser\Lexer;
use PhpParser\ErrorHandler;
use PhpParser\Lexer;
use PhpParser\LexerTest;
use PhpParser\Parser\Php7;
use PhpParser\PhpVersion;
use PhpParser\Token;
require __DIR__ . '/../../../lib/PhpParser/compatibility_tokens.php';
class EmulativeTest extends LexerTest {
protected function getLexer(array $options = []) {
return new Emulative($options);
protected function getLexer() {
return new Emulative();
}
/**
* @dataProvider provideTestReplaceKeywords
*/
public function testReplaceKeywords($keyword, $expectedToken) {
public function testReplaceKeywords(string $keyword, int $expectedToken) {
$lexer = $this->getLexer();
$lexer->startLexing('<?php ' . $keyword);
$this->assertSame($expectedToken, $lexer->getNextToken());
$this->assertSame(0, $lexer->getNextToken());
$code = '<?php ' . $keyword;
$lexer->startLexing($code);
$this->assertEquals([
new Token(\T_OPEN_TAG, '<?php ', 1, 0),
new Token($expectedToken, $keyword, 1, 6),
new Token(0, "\0", 1, \strlen($code)),
], $lexer->getTokens());
}
/**
* @dataProvider provideTestReplaceKeywords
*/
public function testReplaceKeywordsUppercase($keyword, $expectedToken) {
public function testReplaceKeywordsUppercase(string $keyword, int $expectedToken) {
$lexer = $this->getLexer();
$lexer->startLexing('<?php ' . strtoupper($keyword));
$code = '<?php ' . strtoupper($keyword);
$lexer->startLexing($code);
$this->assertSame($expectedToken, $lexer->getNextToken());
$this->assertSame(0, $lexer->getNextToken());
$this->assertEquals([
new Token(\T_OPEN_TAG, '<?php ', 1, 0),
new Token($expectedToken, \strtoupper($keyword), 1, 6),
new Token(0, "\0", 1, \strlen($code)),
], $lexer->getTokens());
}
/**
@ -40,11 +50,15 @@ class EmulativeTest extends LexerTest {
*/
public function testNoReplaceKeywordsAfterObjectOperator(string $keyword) {
$lexer = $this->getLexer();
$lexer->startLexing('<?php ->' . $keyword);
$code = '<?php ->' . $keyword;
$lexer->startLexing($code);
$this->assertSame(\T_OBJECT_OPERATOR, $lexer->getNextToken());
$this->assertSame(\T_STRING, $lexer->getNextToken());
$this->assertSame(0, $lexer->getNextToken());
$this->assertEquals([
new Token(\T_OPEN_TAG, '<?php ', 1, 0),
new Token(\T_OBJECT_OPERATOR, '->', 1, 6),
new Token(\T_STRING, $keyword, 1, 8),
new Token(0, "\0", 1, \strlen($code)),
], $lexer->getTokens());
}
/**
@ -52,11 +66,16 @@ class EmulativeTest extends LexerTest {
*/
public function testNoReplaceKeywordsAfterObjectOperatorWithSpaces(string $keyword) {
$lexer = $this->getLexer();
$lexer->startLexing('<?php -> ' . $keyword);
$code = '<?php -> ' . $keyword;
$lexer->startLexing($code);
$this->assertSame(\T_OBJECT_OPERATOR, $lexer->getNextToken());
$this->assertSame(\T_STRING, $lexer->getNextToken());
$this->assertSame(0, $lexer->getNextToken());
$this->assertEquals([
new Token(\T_OPEN_TAG, '<?php ', 1, 0),
new Token(\T_OBJECT_OPERATOR, '->', 1, 6),
new Token(\T_WHITESPACE, ' ', 1, 8),
new Token(\T_STRING, $keyword, 1, 12),
new Token(0, "\0", 1, \strlen($code)),
], $lexer->getTokens());
}
/**
@ -64,11 +83,15 @@ class EmulativeTest extends LexerTest {
*/
public function testNoReplaceKeywordsAfterNullsafeObjectOperator(string $keyword) {
$lexer = $this->getLexer();
$lexer->startLexing('<?php ?->' . $keyword);
$code = '<?php ?->' . $keyword;
$lexer->startLexing($code);
$this->assertSame(\T_NULLSAFE_OBJECT_OPERATOR, $lexer->getNextToken());
$this->assertSame(\T_STRING, $lexer->getNextToken());
$this->assertSame(0, $lexer->getNextToken());
$this->assertEquals([
new Token(\T_OPEN_TAG, '<?php ', 1, 0),
new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', 1, 6),
new Token(\T_STRING, $keyword, 1, 9),
new Token(0, "\0", 1, \strlen($code)),
], $lexer->getTokens());
}
public function provideTestReplaceKeywords() {
@ -99,8 +122,11 @@ class EmulativeTest extends LexerTest {
private function assertSameTokens(array $expectedTokens, Lexer $lexer) {
$tokens = [];
while (0 !== $token = $lexer->getNextToken($text)) {
$tokens[] = [$token, $text];
foreach ($lexer->getTokens() as $token) {
if ($token->id === 0 || $token->isIgnorable()) {
continue;
}
$tokens[] = [$token->id, $token->text];
}
$this->assertSame($expectedTokens, $tokens);
}
@ -108,7 +134,7 @@ class EmulativeTest extends LexerTest {
/**
* @dataProvider provideTestLexNewFeatures
*/
public function testLexNewFeatures($code, array $expectedTokens) {
public function testLexNewFeatures(string $code, array $expectedTokens) {
$lexer = $this->getLexer();
$lexer->startLexing('<?php ' . $code);
$this->assertSameTokens($expectedTokens, $lexer);
@ -117,15 +143,18 @@ class EmulativeTest extends LexerTest {
/**
* @dataProvider provideTestLexNewFeatures
*/
public function testLeaveStuffAloneInStrings($code) {
public function testLeaveStuffAloneInStrings(string $code) {
$stringifiedToken = '"' . addcslashes($code, '"\\') . '"';
$lexer = $this->getLexer();
$lexer->startLexing('<?php ' . $stringifiedToken);
$fullCode = '<?php ' . $stringifiedToken;
$lexer->startLexing($fullCode);
$this->assertSame(\T_CONSTANT_ENCAPSED_STRING, $lexer->getNextToken($text));
$this->assertSame($stringifiedToken, $text);
$this->assertSame(0, $lexer->getNextToken());
$this->assertEquals([
new Token(\T_OPEN_TAG, '<?php ', 1, 0),
new Token(\T_CONSTANT_ENCAPSED_STRING, $stringifiedToken, 1, 6),
new Token(0, "\0", \substr_count($fullCode, "\n") + 1, \strlen($fullCode)),
], $lexer->getTokens());
}
/**
@ -375,7 +404,7 @@ class EmulativeTest extends LexerTest {
* @dataProvider provideTestTargetVersion
*/
public function testTargetVersion(string $phpVersion, string $code, array $expectedTokens) {
$lexer = $this->getLexer(['phpVersion' => $phpVersion]);
$lexer = new Emulative(PhpVersion::fromString($phpVersion));
$lexer->startLexing('<?php ' . $code);
$this->assertSameTokens($expectedTokens, $lexer);
}

226
test/PhpParser/LexerTest.php
View File

@ -6,8 +6,8 @@ require __DIR__ . '/../../lib/PhpParser/compatibility_tokens.php';
class LexerTest extends \PHPUnit\Framework\TestCase {
/* To allow overwriting in parent class */
protected function getLexer(array $options = []) {
return new Lexer($options);
protected function getLexer() {
return new Lexer();
}
/**
@ -19,9 +19,7 @@ class LexerTest extends \PHPUnit\Framework\TestCase {
}
$errorHandler = new ErrorHandler\Collecting();
$lexer = $this->getLexer(['usedAttributes' => [
'comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'
]]);
$lexer = $this->getLexer();
$lexer->startLexing($code, $errorHandler);
$errors = $errorHandler->getErrors();
@ -52,230 +50,54 @@ class LexerTest extends \PHPUnit\Framework\TestCase {
$this->expectExceptionMessage('Unterminated comment on line 1');
$lexer = $this->getLexer();
$lexer->startLexing("<?php readonly /*");
$lexer->getNextToken();
}
/**
* @dataProvider provideTestLex
*/
public function testLex($code, $options, $tokens) {
$lexer = $this->getLexer($options);
public function testLex($code, $expectedTokens) {
$lexer = $this->getLexer();
$lexer->startLexing($code);
while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
$token = array_shift($tokens);
$tokens = $lexer->getTokens();
foreach ($tokens as $token) {
if ($token->id === 0 || $token->isIgnorable()) {
continue;
}
$this->assertSame($token[0], $id);
$this->assertSame($token[1], $value);
$this->assertEquals($token[2], $startAttributes);
$this->assertEquals($token[3], $endAttributes);
$expectedToken = array_shift($expectedTokens);
$this->assertSame($expectedToken[0], $token->id);
$this->assertSame($expectedToken[1], $token->text);
}
}
public function provideTestLex() {
return [
// tests conversion of closing PHP tag and drop of whitespace and opening tags
[
'<?php tokens ?>plaintext',
[],
[
[
\T_STRING, 'tokens',
['startLine' => 1], ['endLine' => 1]
],
[
\T_CLOSE_TAG, '?>',
['startLine' => 1], ['endLine' => 1]
],
[
\T_INLINE_HTML, 'plaintext',
['startLine' => 1, 'hasLeadingNewline' => false],
['endLine' => 1]
],
]
],
// tests line numbers
[
'<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $',
[],
[
[
ord('$'), '$',
['startLine' => 2], ['endLine' => 2]
],
[
\T_STRING, 'token',
['startLine' => 2], ['endLine' => 2]
],
[
ord('$'), '$',
[
'startLine' => 3,
'comments' => [
new Comment\Doc('/** doc' . "\n" . 'comment */',
2, 14, 5,
3, 31, 5),
]
],
['endLine' => 3]
],
]
],
// tests comment extraction
[
'<?php /* comment */ // comment' . "\n" . '/** docComment 1 *//** docComment 2 */ token',
[],
[
[
\T_STRING, 'token',
[
'startLine' => 2,
'comments' => [
new Comment('/* comment */',
1, 6, 1, 1, 18, 1),
new Comment('// comment',
1, 20, 3, 1, 29, 3),
new Comment\Doc('/** docComment 1 */',
2, 31, 5, 2, 49, 5),
new Comment\Doc('/** docComment 2 */',
2, 50, 6, 2, 68, 6),
],
],
['endLine' => 2]
],
]
],
// tests differing start and end line
[
'<?php "foo' . "\n" . 'bar"',
[],
[
[
\T_CONSTANT_ENCAPSED_STRING, '"foo' . "\n" . 'bar"',
['startLine' => 1], ['endLine' => 2]
],
]
],
// tests exact file offsets
[
'<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
['usedAttributes' => ['startFilePos', 'endFilePos']],
[
[
\T_CONSTANT_ENCAPSED_STRING, '"a"',
['startFilePos' => 6], ['endFilePos' => 8]
],
[
ord(';'), ';',
['startFilePos' => 9], ['endFilePos' => 9]
],
[
\T_CONSTANT_ENCAPSED_STRING, '"b"',
['startFilePos' => 18], ['endFilePos' => 20]
],
[
ord(';'), ';',
['startFilePos' => 21], ['endFilePos' => 21]
],
]
],
// tests token offsets
[
'<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
['usedAttributes' => ['startTokenPos', 'endTokenPos']],
[
[
\T_CONSTANT_ENCAPSED_STRING, '"a"',
['startTokenPos' => 1], ['endTokenPos' => 1]
],
[
ord(';'), ';',
['startTokenPos' => 2], ['endTokenPos' => 2]
],
[
\T_CONSTANT_ENCAPSED_STRING, '"b"',
['startTokenPos' => 6], ['endTokenPos' => 6]
],
[
ord(';'), ';',
['startTokenPos' => 7], ['endTokenPos' => 7]
],
]
],
// tests all attributes being disabled
[
'<?php /* foo */ $bar;',
['usedAttributes' => []],
[
[
\T_VARIABLE, '$bar',
[], []
],
[
ord(';'), ';',
[], []
]
]
],
// tests no tokens
[
'',
[],
[]
],
// tests PHP 8 T_NAME_* emulation
[
'<?php Foo\Bar \Foo\Bar namespace\Foo\Bar Foo\Bar\\',
['usedAttributes' => []],
[
[\T_NAME_QUALIFIED, 'Foo\Bar', [], []],
[\T_NAME_FULLY_QUALIFIED, '\Foo\Bar', [], []],
[\T_NAME_RELATIVE, 'namespace\Foo\Bar', [], []],
[\T_NAME_QUALIFIED, 'Foo\Bar', [], []],
[\T_NS_SEPARATOR, '\\', [], []],
[\T_NAME_QUALIFIED, 'Foo\Bar'],
[\T_NAME_FULLY_QUALIFIED, '\Foo\Bar'],
[\T_NAME_RELATIVE, 'namespace\Foo\Bar'],
[\T_NAME_QUALIFIED, 'Foo\Bar'],
[\T_NS_SEPARATOR, '\\'],
]
],
// tests PHP 8 T_NAME_* emulation with reserved keywords
[
'<?php fn\use \fn\use namespace\fn\use fn\use\\',
['usedAttributes' => []],
[
[\T_NAME_QUALIFIED, 'fn\use', [], []],
[\T_NAME_FULLY_QUALIFIED, '\fn\use', [], []],
[\T_NAME_RELATIVE, 'namespace\fn\use', [], []],
[\T_NAME_QUALIFIED, 'fn\use', [], []],
[\T_NS_SEPARATOR, '\\', [], []],
[\T_NAME_QUALIFIED, 'fn\use'],
[\T_NAME_FULLY_QUALIFIED, '\fn\use'],
[\T_NAME_RELATIVE, 'namespace\fn\use'],
[\T_NAME_QUALIFIED, 'fn\use'],
[\T_NS_SEPARATOR, '\\'],
]
],
];
}
/**
* @dataProvider provideTestHaltCompiler
*/
public function testHandleHaltCompiler($code, $remaining) {
$lexer = $this->getLexer();
$lexer->startLexing($code);
while (\T_HALT_COMPILER !== $lexer->getNextToken());
$lexer->getNextToken();
$lexer->getNextToken();
$lexer->getNextToken();
$this->assertSame($remaining, $lexer->handleHaltCompiler());
$this->assertSame(0, $lexer->getNextToken());
}
public function provideTestHaltCompiler() {
return [
['<?php ... __halt_compiler();', ''],
['<?php ... __halt_compiler();Remaining Text', 'Remaining Text'],
['<?php ... __halt_compiler ( ) ;Remaining Text', 'Remaining Text'],
['<?php ... __halt_compiler() ?>Remaining Text', 'Remaining Text'],
['<?php ... __halt_compiler();' . "\0", "\0"],
['<?php ... __halt_compiler /* */ ( ) ;Remaining Text', 'Remaining Text'],
];
}
public function testGetTokens() {
$code = '<?php "a";' . "\n" . '// foo' . "\n" . '// bar' . "\n\n" . '"b";';
$expectedTokens = [

116
test/PhpParser/NodeAbstractTest.php
View File

@ -220,7 +220,11 @@ PHP;
"name": "functionName",
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 7,
"startFilePos": 45,
"endLine": 4,
"endTokenPos": 7,
"endFilePos": 56
}
},
"params": [
@ -234,7 +238,11 @@ PHP;
"name": "a",
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 10,
"startFilePos": 59,
"endLine": 4,
"endTokenPos": 10,
"endFilePos": 60
}
},
"default": {
@ -242,7 +250,11 @@ PHP;
"value": 0,
"attributes": {
"startLine": 4,
"startTokenPos": 14,
"startFilePos": 64,
"endLine": 4,
"endTokenPos": 14,
"endFilePos": 64,
"rawValue": "0",
"kind": 10
}
@ -251,7 +263,11 @@ PHP;
"attrGroups": [],
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 9,
"startFilePos": 58,
"endLine": 4,
"endTokenPos": 14,
"endFilePos": 64
}
},
{
@ -264,7 +280,11 @@ PHP;
"name": "b",
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 17,
"startFilePos": 67,
"endLine": 4,
"endTokenPos": 17,
"endFilePos": 68
}
},
"default": {
@ -272,7 +292,11 @@ PHP;
"value": 1,
"attributes": {
"startLine": 4,
"startTokenPos": 21,
"startFilePos": 72,
"endLine": 4,
"endTokenPos": 21,
"endFilePos": 74,
"rawValue": "1.0"
}
},
@ -280,7 +304,11 @@ PHP;
"attrGroups": [],
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 17,
"startFilePos": 67,
"endLine": 4,
"endTokenPos": 21,
"endFilePos": 74
}
}
],
@ -294,7 +322,11 @@ PHP;
"value": "Foo",
"attributes": {
"startLine": 5,
"startTokenPos": 28,
"startFilePos": 88,
"endLine": 5,
"endTokenPos": 28,
"endFilePos": 92,
"kind": 1,
"rawValue": "'Foo'"
}
@ -302,13 +334,23 @@ PHP;
],
"attributes": {
"startLine": 5,
"endLine": 5
"startTokenPos": 26,
"startFilePos": 83,
"endLine": 5,
"endTokenPos": 29,
"endFilePos": 93
}
}
],
"attrGroups": [],
"namespacedName": null,
"attributes": {
"startLine": 4,
"startTokenPos": 5,
"startFilePos": 36,
"endLine": 6,
"endTokenPos": 31,
"endFilePos": 95,
"comments": [
{
"nodeType": "Comment",
@ -330,9 +372,7 @@ PHP;
"endFilePos": 34,
"endTokenPos": 3
}
],
"startLine": 4,
"endLine": 6
]
}
}
]
@ -342,6 +382,12 @@ JSON;
{
"nodeType": "Stmt_Function",
"attributes": {
"startLine": 4,
"startTokenPos": 5,
"startFilePos": 36,
"endLine": 6,
"endTokenPos": 31,
"endFilePos": 95,
"comments": [
{
"nodeType": "Comment",
@ -363,16 +409,18 @@ JSON;
"endFilePos": 34,
"endTokenPos": 3
}
],
"startLine": 4,
"endLine": 6
]
},
"byRef": false,
"name": {
"nodeType": "Identifier",
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 7,
"startFilePos": 45,
"endLine": 4,
"endTokenPos": 7,
"endFilePos": 56
},
"name": "functionName"
},
@ -381,7 +429,11 @@ JSON;
"nodeType": "Param",
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 9,
"startFilePos": 58,
"endLine": 4,
"endTokenPos": 14,
"endFilePos": 64
},
"type": null,
"byRef": true,
@ -390,7 +442,11 @@ JSON;
"nodeType": "Expr_Variable",
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 10,
"startFilePos": 59,
"endLine": 4,
"endTokenPos": 10,
"endFilePos": 60
},
"name": "a"
},
@ -398,7 +454,11 @@ JSON;
"nodeType": "Scalar_Int",
"attributes": {
"startLine": 4,
"startTokenPos": 14,
"startFilePos": 64,
"endLine": 4,
"endTokenPos": 14,
"endFilePos": 64,
"rawValue": "0",
"kind": 10
},
@ -411,7 +471,11 @@ JSON;
"nodeType": "Param",
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 17,
"startFilePos": 67,
"endLine": 4,
"endTokenPos": 21,
"endFilePos": 74
},
"type": null,
"byRef": false,
@ -420,7 +484,11 @@ JSON;
"nodeType": "Expr_Variable",
"attributes": {
"startLine": 4,
"endLine": 4
"startTokenPos": 17,
"startFilePos": 67,
"endLine": 4,
"endTokenPos": 17,
"endFilePos": 68
},
"name": "b"
},
@ -428,7 +496,11 @@ JSON;
"nodeType": "Scalar_Float",
"attributes": {
"startLine": 4,
"startTokenPos": 21,
"startFilePos": 72,
"endLine": 4,
"endTokenPos": 21,
"endFilePos": 74,
"rawValue": "1.0"
},
"value": 1
@ -443,14 +515,22 @@ JSON;
"nodeType": "Stmt_Echo",
"attributes": {
"startLine": 5,
"endLine": 5
"startTokenPos": 26,
"startFilePos": 83,
"endLine": 5,
"endTokenPos": 29,
"endFilePos": 93
},
"exprs": [
{
"nodeType": "Scalar_String",
"attributes": {
"startLine": 5,
"startTokenPos": 28,
"startFilePos": 88,
"endLine": 5,
"endTokenPos": 28,
"endFilePos": 92,
"kind": 1,
"rawValue": "'Foo'"
},

5
test/PhpParser/NodeDumperTest.php
View File

@ -58,10 +58,7 @@ class NodeDumperTest extends \PHPUnit\Framework\TestCase {
}
public function testDumpWithPositions() {
$parser = (new ParserFactory())->create(
ParserFactory::ONLY_PHP7,
new Lexer(['usedAttributes' => ['startLine', 'endLine', 'startFilePos', 'endFilePos']])
);
$parser = (new ParserFactory())->createForHostVersion();
$dumper = new NodeDumper(['dumpPositions' => true]);
$code = "<?php\n\$a = 1;\necho \$a;";

22
test/PhpParser/ParserTest.php
View File

@ -33,12 +33,7 @@ abstract class ParserTest extends \PHPUnit\Framework\TestCase {
}
public function testAttributeAssignment() {
$lexer = new Lexer([
'usedAttributes' => [
'comments', 'startLine', 'endLine',
'startTokenPos', 'endTokenPos',
]
]);
$lexer = new Lexer();
$code = <<<'EOC'
<?php
@ -66,6 +61,8 @@ EOC;
'endLine' => 7,
'startTokenPos' => 3,
'endTokenPos' => 21,
'startFilePos' => 25,
'endFilePos' => 86,
], $fn->getAttributes());
$param = $fn->params[0];
@ -75,6 +72,8 @@ EOC;
'endLine' => 3,
'startTokenPos' => 7,
'endTokenPos' => 7,
'startFilePos' => 39,
'endFilePos' => 40,
], $param->getAttributes());
/** @var Stmt\Echo_ $echo */
@ -91,6 +90,8 @@ EOC;
'endLine' => 6,
'startTokenPos' => 16,
'endTokenPos' => 19,
'startFilePos' => 77,
'endFilePos' => 84,
], $echo->getAttributes());
/** @var \PhpParser\Node\Expr\Variable $var */
@ -101,6 +102,8 @@ EOC;
'endLine' => 6,
'startTokenPos' => 18,
'endTokenPos' => 18,
'startFilePos' => 82,
'endFilePos' => 83,
], $var->getAttributes());
}
@ -194,8 +197,9 @@ EOC;
}
class InvalidTokenLexer extends Lexer {
public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null): int {
$value = 'foobar';
return 999;
public function startLexing(string $code, ?ErrorHandler $errorHandler = null): void {
$this->tokens = [
new Token(999, 'foobar', 42),
];
}
}

17
test/PhpParser/PrettyPrinterTest.php
View File

@ -184,14 +184,7 @@ class PrettyPrinterTest extends CodeTestAbstract {
* @covers \PhpParser\PrettyPrinter\Standard<extended>
*/
public function testFormatPreservingPrint($name, $code, $modification, $expected, $modeLine) {
$lexer = new Lexer\Emulative([
'usedAttributes' => [
'comments',
'startLine', 'endLine',
'startTokenPos', 'endTokenPos',
],
]);
$lexer = new Lexer\Emulative();
$parser = new Parser\Php7($lexer);
$traverser = new NodeTraverser(new NodeVisitor\CloningVisitor());
@ -233,13 +226,7 @@ CODE
* the pretty printer tests (i.e. returns the input if no changes occurred).
*/
$lexer = new Lexer\Emulative([
'usedAttributes' => [
'comments',
'startLine', 'endLine',
'startTokenPos', 'endTokenPos',
],
]);
$lexer = new Lexer\Emulative();
$parser = new Php7($lexer);

7
test_old/run.php
View File

@ -127,12 +127,7 @@ switch ($testType) {
showHelp('Test type must be one of: PHP or Symfony');
}
$lexer = new PhpParser\Lexer\Emulative([
'usedAttributes' => [
'comments', 'startLine', 'endLine', 'startTokenPos', 'endTokenPos',
],
'phpVersion' => $phpVersion,
]);
$lexer = new PhpParser\Lexer\Emulative(\PhpParser\PhpVersion::fromString($phpVersion));
if (version_compare($phpVersion, '7.0', '>=')) {
$parser = new PhpParser\Parser\Php7($lexer);
} else {

4
tools/fuzzing/target.php
View File

@ -21,9 +21,7 @@ if (!file_exists($autoload)) {
require $autoload;
$lexer = new PhpParser\Lexer([
'usedAttributes' => ['comments', 'startLine', 'endLine', 'startTokenPos'],
]);
$lexer = new PhpParser\Lexer();
$parser = new PhpParser\Parser\Php7($lexer);
$prettyPrinter = new PhpParser\PrettyPrinter\Standard();
$nodeDumper = new PhpParser\NodeDumper();