2017-11-02 18:56:01 +01:00
|
|
|
<?php declare(strict_types=1);
|
2011-07-13 12:24:10 +02:00
|
|
|
|
2014-02-06 14:44:16 +01:00
|
|
|
namespace PhpParser;
|
|
|
|
|
2022-06-19 18:05:52 +02:00
|
|
|
require __DIR__ . '/../../lib/PhpParser/compatibility_tokens.php';
|
2015-06-13 18:39:55 +02:00
|
|
|
|
2022-08-29 21:52:53 +02:00
|
|
|
class LexerTest extends \PHPUnit\Framework\TestCase {
|
2014-11-27 20:57:38 +01:00
|
|
|
/* To allow overwriting in parent class */
|
2023-07-29 15:10:11 +02:00
|
|
|
protected function getLexer() {
|
|
|
|
return new Lexer();
|
2014-11-27 20:57:38 +01:00
|
|
|
}
|
2012-04-25 20:04:46 +02:00
|
|
|
|
2011-07-13 12:24:10 +02:00
|
|
|
/**
|
|
|
|
* @dataProvider provideTestError
|
|
|
|
*/
|
2016-09-30 18:28:35 +02:00
|
|
|
public function testError($code, $messages) {
|
2015-02-26 19:19:40 +01:00
|
|
|
if (defined('HHVM_VERSION')) {
|
|
|
|
$this->markTestSkipped('HHVM does not throw warnings from token_get_all()');
|
|
|
|
}
|
|
|
|
|
2016-10-09 13:15:24 +02:00
|
|
|
$errorHandler = new ErrorHandler\Collecting();
|
2023-07-29 15:10:11 +02:00
|
|
|
$lexer = $this->getLexer();
|
2023-08-13 16:03:26 +02:00
|
|
|
$lexer->tokenize($code, $errorHandler);
|
2016-10-09 13:15:24 +02:00
|
|
|
$errors = $errorHandler->getErrors();
|
2011-07-13 12:24:10 +02:00
|
|
|
|
2017-12-12 09:02:49 -02:00
|
|
|
$this->assertCount(count($messages), $errors);
|
2016-09-30 18:28:35 +02:00
|
|
|
for ($i = 0; $i < count($messages); $i++) {
|
|
|
|
$this->assertSame($messages[$i], $errors[$i]->getMessageWithColumnInfo($code));
|
2011-07-13 12:24:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
public function provideTestError() {
|
2017-08-13 14:35:03 +02:00
|
|
|
return [
|
|
|
|
["<?php /*", ["Unterminated comment from 1:7 to 1:9"]],
|
2020-07-22 18:35:11 +02:00
|
|
|
["<?php /*\n", ["Unterminated comment from 1:7 to 2:1"]],
|
2017-08-13 14:35:03 +02:00
|
|
|
["<?php \1", ["Unexpected character \"\1\" (ASCII 1) from 1:7 to 1:7"]],
|
|
|
|
["<?php \0", ["Unexpected null byte from 1:7 to 1:7"]],
|
2016-09-30 18:28:35 +02:00
|
|
|
// Error with potentially emulated token
|
2017-08-13 14:35:03 +02:00
|
|
|
["<?php ?? \0", ["Unexpected null byte from 1:10 to 1:10"]],
|
|
|
|
["<?php\n\0\1 foo /* bar", [
|
2016-09-30 18:28:35 +02:00
|
|
|
"Unexpected null byte from 2:1 to 2:1",
|
|
|
|
"Unexpected character \"\1\" (ASCII 1) from 2:2 to 2:2",
|
|
|
|
"Unterminated comment from 2:8 to 2:14"
|
2017-08-13 14:35:03 +02:00
|
|
|
]],
|
|
|
|
];
|
2011-07-13 12:24:10 +02:00
|
|
|
}
|
|
|
|
|
2023-05-20 22:26:53 +02:00
|
|
|
public function testDefaultErrorHandler() {
|
|
|
|
$this->expectException(Error::class);
|
|
|
|
$this->expectExceptionMessage('Unterminated comment on line 1');
|
|
|
|
$lexer = $this->getLexer();
|
2023-08-13 16:03:26 +02:00
|
|
|
$lexer->tokenize("<?php readonly /*");
|
2023-05-20 22:26:53 +02:00
|
|
|
}
|
|
|
|
|
2011-07-13 12:24:10 +02:00
|
|
|
/**
|
|
|
|
* @dataProvider provideTestLex
|
|
|
|
*/
|
2023-07-29 15:10:11 +02:00
|
|
|
public function testLex($code, $expectedTokens) {
|
|
|
|
$lexer = $this->getLexer();
|
2023-08-13 16:03:26 +02:00
|
|
|
$tokens = $lexer->tokenize($code);
|
2023-07-29 15:10:11 +02:00
|
|
|
foreach ($tokens as $token) {
|
|
|
|
if ($token->id === 0 || $token->isIgnorable()) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
$expectedToken = array_shift($expectedTokens);
|
2011-07-13 12:24:10 +02:00
|
|
|
|
2023-07-29 15:10:11 +02:00
|
|
|
$this->assertSame($expectedToken[0], $token->id);
|
|
|
|
$this->assertSame($expectedToken[1], $token->text);
|
2011-07-13 12:24:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
public function provideTestLex() {
|
2017-08-13 14:35:03 +02:00
|
|
|
return [
|
2020-07-23 12:28:13 +02:00
|
|
|
// tests PHP 8 T_NAME_* emulation
|
|
|
|
[
|
|
|
|
'<?php Foo\Bar \Foo\Bar namespace\Foo\Bar Foo\Bar\\',
|
|
|
|
[
|
2023-07-29 15:10:11 +02:00
|
|
|
[\T_NAME_QUALIFIED, 'Foo\Bar'],
|
|
|
|
[\T_NAME_FULLY_QUALIFIED, '\Foo\Bar'],
|
|
|
|
[\T_NAME_RELATIVE, 'namespace\Foo\Bar'],
|
|
|
|
[\T_NAME_QUALIFIED, 'Foo\Bar'],
|
|
|
|
[\T_NS_SEPARATOR, '\\'],
|
2020-08-09 21:11:49 +02:00
|
|
|
]
|
|
|
|
],
|
|
|
|
// tests PHP 8 T_NAME_* emulation with reserved keywords
|
|
|
|
[
|
|
|
|
'<?php fn\use \fn\use namespace\fn\use fn\use\\',
|
|
|
|
[
|
2023-07-29 15:10:11 +02:00
|
|
|
[\T_NAME_QUALIFIED, 'fn\use'],
|
|
|
|
[\T_NAME_FULLY_QUALIFIED, '\fn\use'],
|
|
|
|
[\T_NAME_RELATIVE, 'namespace\fn\use'],
|
|
|
|
[\T_NAME_QUALIFIED, 'fn\use'],
|
|
|
|
[\T_NS_SEPARATOR, '\\'],
|
2020-07-23 12:28:13 +02:00
|
|
|
]
|
|
|
|
],
|
2017-08-13 14:35:03 +02:00
|
|
|
];
|
2011-07-13 12:24:10 +02:00
|
|
|
}
|
|
|
|
|
2014-12-18 23:26:17 +01:00
|
|
|
public function testGetTokens() {
|
2020-06-27 18:53:09 +02:00
|
|
|
$code = '<?php "a";' . "\n" . '// foo' . "\n" . '// bar' . "\n\n" . '"b";';
|
2017-08-13 14:35:03 +02:00
|
|
|
$expectedTokens = [
|
2022-06-04 17:34:48 +02:00
|
|
|
new Token(T_OPEN_TAG, '<?php ', 1, 0),
|
|
|
|
new Token(T_CONSTANT_ENCAPSED_STRING, '"a"', 1, 6),
|
|
|
|
new Token(\ord(';'), ';', 1, 9),
|
|
|
|
new Token(T_WHITESPACE, "\n", 1, 10),
|
|
|
|
new Token(T_COMMENT, '// foo', 2, 11),
|
|
|
|
new Token(T_WHITESPACE, "\n", 2, 17),
|
|
|
|
new Token(T_COMMENT, '// bar', 3, 18),
|
|
|
|
new Token(T_WHITESPACE, "\n\n", 3, 24),
|
|
|
|
new Token(T_CONSTANT_ENCAPSED_STRING, '"b"', 5, 26),
|
|
|
|
new Token(\ord(';'), ';', 5, 29),
|
|
|
|
new Token(0, "\0", 5, 30),
|
2017-08-13 14:35:03 +02:00
|
|
|
];
|
2014-12-18 23:26:17 +01:00
|
|
|
|
|
|
|
$lexer = $this->getLexer();
|
2023-08-13 16:03:26 +02:00
|
|
|
$this->assertEquals($expectedTokens, $lexer->tokenize($code));
|
2014-12-18 23:26:17 +01:00
|
|
|
}
|
2014-11-27 20:38:14 +01:00
|
|
|
}
|