From 60d025a914aff02d9d7403bbe858e2553fa61f24 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Sat, 11 May 2019 20:01:25 +0200 Subject: [PATCH] Fix attributes for zero-length nop nodes Previously zero-length nop nodes used the lookahead start attributes and current end attributes. This choice ends up being somewhat weird, because the end attributes will be the at the last non-whitespace, non-comment token, which might be quite far back. More problematically, we may not have encountered any non-discarded token if we're at the start of the file, in which case we will have no end attributes to assign. Change things to use a canonical "zero-length" node representation, where the end position (token & file) will be exactly one before the start position. Fixes #589. --- grammar/php5.y | 6 +++--- grammar/php7.y | 6 +++--- grammar/rebuildParsers.php | 18 +++++++++++++++--- lib/PhpParser/Parser/Php5.php | 6 +++--- lib/PhpParser/Parser/Php7.php | 6 +++--- lib/PhpParser/ParserAbstract.php | 20 ++++++++++++++++++++ test/code/parser/nopPositions.test | 13 +++++++++++++ 7 files changed, 60 insertions(+), 15 deletions(-) create mode 100644 test/code/parser/nopPositions.test diff --git a/grammar/php5.y b/grammar/php5.y index e890a410..e775f2ca 100644 --- a/grammar/php5.y +++ b/grammar/php5.y @@ -16,7 +16,7 @@ top_statement_list_ex: top_statement_list: top_statement_list_ex - { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes); + { makeZeroLengthNop($nop, $this->lookaheadStartAttributes); if ($nop !== null) { $1[] = $nop; } $$ = $1; } ; @@ -160,7 +160,7 @@ inner_statement_list_ex: inner_statement_list: inner_statement_list_ex - { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes); + { makeZeroLengthNop($nop, $this->lookaheadStartAttributes); if ($nop !== null) { $1[] = $nop; } $$ = $1; } ; @@ -461,7 +461,7 @@ class_statement_list_ex: class_statement_list: class_statement_list_ex - { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes); + { makeZeroLengthNop($nop, $this->lookaheadStartAttributes); if ($nop !== null) { $1[] = $nop; } $$ = $1; } ; diff --git a/grammar/php7.y b/grammar/php7.y index c089ea51..23b193bb 100644 --- a/grammar/php7.y +++ b/grammar/php7.y @@ -16,7 +16,7 @@ top_statement_list_ex: top_statement_list: top_statement_list_ex - { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes); + { makeZeroLengthNop($nop, $this->lookaheadStartAttributes); if ($nop !== null) { $1[] = $nop; } $$ = $1; } ; @@ -196,7 +196,7 @@ inner_statement_list_ex: inner_statement_list: inner_statement_list_ex - { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes); + { makeZeroLengthNop($nop, $this->lookaheadStartAttributes); if ($nop !== null) { $1[] = $nop; } $$ = $1; } ; @@ -530,7 +530,7 @@ class_statement_list_ex: class_statement_list: class_statement_list_ex - { makeNop($nop, $this->lookaheadStartAttributes, $this->endAttributes); + { makeZeroLengthNop($nop, $this->lookaheadStartAttributes); if ($nop !== null) { $1[] = $nop; } $$ = $1; } ; diff --git a/grammar/rebuildParsers.php b/grammar/rebuildParsers.php index 1882d5fe..b913cdac 100644 --- a/grammar/rebuildParsers.php +++ b/grammar/rebuildParsers.php @@ -13,9 +13,12 @@ $tmpResultFile = __DIR__ . '/tmp_parser.php'; $resultDir = __DIR__ . '/../lib/PhpParser/Parser'; $tokensResultsFile = $resultDir . '/Tokens.php'; -// check for kmyacc.exe binary in this directory, otherwise fall back to global name -$kmyacc = __DIR__ . '/kmyacc.exe'; -if (!file_exists($kmyacc)) { +// check for kmyacc binary in this directory, otherwise fall back to global name +if (file_exists(__DIR__ . '/kmyacc.exe')) { + $kmyacc = __DIR__ . '/kmyacc.exe'; +} else if (file_exists(__DIR__ . '/kmyacc')) { + $kmyacc = __DIR__ . '/kmyacc'; +} else { $kmyacc = 'kmyacc'; } @@ -175,6 +178,15 @@ function resolveMacros($code) { . ' else { ' . $args[0] . ' = null; }'; } + if ('makeZeroLengthNop' == $name) { + assertArgs(2, $args, $name); + + return '$startAttributes = ' . $args[1] . ';' + . ' if (isset($startAttributes[\'comments\']))' + . ' { ' . $args[0] . ' = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); }' + . ' else { ' . $args[0] . ' = null; }'; + } + if ('strKind' == $name) { assertArgs(1, $args, $name); diff --git a/lib/PhpParser/Parser/Php5.php b/lib/PhpParser/Parser/Php5.php index 9099052a..a558d28b 100644 --- a/lib/PhpParser/Parser/Php5.php +++ b/lib/PhpParser/Parser/Php5.php @@ -944,7 +944,7 @@ class Php5 extends \PhpParser\ParserAbstract $this->semValue = array(); }, 4 => function ($stackPos) { - $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($startAttributes + $this->endAttributes); } else { $nop = null; }; + $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); } else { $nop = null; }; if ($nop !== null) { $this->semStack[$stackPos-(1-1)][] = $nop; } $this->semValue = $this->semStack[$stackPos-(1-1)]; }, 5 => function ($stackPos) { @@ -1317,7 +1317,7 @@ class Php5 extends \PhpParser\ParserAbstract $this->semValue = array(); }, 126 => function ($stackPos) { - $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($startAttributes + $this->endAttributes); } else { $nop = null; }; + $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); } else { $nop = null; }; if ($nop !== null) { $this->semStack[$stackPos-(1-1)][] = $nop; } $this->semValue = $this->semStack[$stackPos-(1-1)]; }, 127 => function ($stackPos) { @@ -1715,7 +1715,7 @@ class Php5 extends \PhpParser\ParserAbstract $this->semValue = array(); }, 255 => function ($stackPos) { - $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($startAttributes + $this->endAttributes); } else { $nop = null; }; + $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); } else { $nop = null; }; if ($nop !== null) { $this->semStack[$stackPos-(1-1)][] = $nop; } $this->semValue = $this->semStack[$stackPos-(1-1)]; }, 256 => function ($stackPos) { diff --git a/lib/PhpParser/Parser/Php7.php b/lib/PhpParser/Parser/Php7.php index 18f4d779..75c8305f 100644 --- a/lib/PhpParser/Parser/Php7.php +++ b/lib/PhpParser/Parser/Php7.php @@ -869,7 +869,7 @@ class Php7 extends \PhpParser\ParserAbstract $this->semValue = array(); }, 4 => function ($stackPos) { - $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($startAttributes + $this->endAttributes); } else { $nop = null; }; + $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); } else { $nop = null; }; if ($nop !== null) { $this->semStack[$stackPos-(1-1)][] = $nop; } $this->semValue = $this->semStack[$stackPos-(1-1)]; }, 5 => function ($stackPos) { @@ -1275,7 +1275,7 @@ class Php7 extends \PhpParser\ParserAbstract $this->semValue = array(); }, 137 => function ($stackPos) { - $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($startAttributes + $this->endAttributes); } else { $nop = null; }; + $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); } else { $nop = null; }; if ($nop !== null) { $this->semStack[$stackPos-(1-1)][] = $nop; } $this->semValue = $this->semStack[$stackPos-(1-1)]; }, 138 => function ($stackPos) { @@ -1694,7 +1694,7 @@ class Php7 extends \PhpParser\ParserAbstract $this->semValue = array(); }, 273 => function ($stackPos) { - $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($startAttributes + $this->endAttributes); } else { $nop = null; }; + $startAttributes = $this->lookaheadStartAttributes; if (isset($startAttributes['comments'])) { $nop = new Stmt\Nop($this->createZeroLengthAttributes($startAttributes)); } else { $nop = null; }; if ($nop !== null) { $this->semStack[$stackPos-(1-1)][] = $nop; } $this->semValue = $this->semStack[$stackPos-(1-1)]; }, 274 => function ($stackPos) { diff --git a/lib/PhpParser/ParserAbstract.php b/lib/PhpParser/ParserAbstract.php index 4d95a6f6..975e17c6 100644 --- a/lib/PhpParser/ParserAbstract.php +++ b/lib/PhpParser/ParserAbstract.php @@ -839,6 +839,26 @@ abstract class ParserAbstract implements Parser } } + /** + * Create attributes for a zero-length node with the given start attributes. + * + * @param array $startAttributes + * @return array + */ + protected function createZeroLengthAttributes(array $startAttributes) { + $attributes = $startAttributes; + if (isset($startAttributes['startLine'])) { + $attributes['endLine'] = $startAttributes['startLine']; + } + if (isset($startAttributes['startTokenPos'])) { + $attributes['endTokenPos'] = $startAttributes['startTokenPos'] - 1; + } + if (isset($startAttributes['startFilePos'])) { + $attributes['endFilePos'] = $startAttributes['startFilePos'] - 1; + } + return $attributes; + } + protected function checkModifier($a, $b, $modifierPos) { // Jumping through some hoops here because verifyModifier() is also used elsewhere try { diff --git a/test/code/parser/nopPositions.test b/test/code/parser/nopPositions.test new file mode 100644 index 00000000..a981329b --- /dev/null +++ b/test/code/parser/nopPositions.test @@ -0,0 +1,13 @@ +Positions for leading nop statement +----- +