xref: /PHP-Parser/test/PhpParser/LexerTest.php (revision daaadc3b) 
1<?php declare(strict_types=1);
2
3namespace PhpParser;
4
5require __DIR__ . '/../../lib/PhpParser/compatibility_tokens.php';
6
7class LexerTest extends \PHPUnit\Framework\TestCase {
8    /* To allow overwriting in parent class */
9    protected function getLexer() {
10        return new Lexer();
11    }
12
13    /**
14     * @dataProvider provideTestError
15     */
16    public function testError($code, $messages): void {
17        if (defined('HHVM_VERSION')) {
18            $this->markTestSkipped('HHVM does not throw warnings from token_get_all()');
19        }
20
21        $errorHandler = new ErrorHandler\Collecting();
22        $lexer = $this->getLexer();
23        $lexer->tokenize($code, $errorHandler);
24        $errors = $errorHandler->getErrors();
25
26        $this->assertCount(count($messages), $errors);
27        for ($i = 0; $i < count($messages); $i++) {
28            $this->assertSame($messages[$i], $errors[$i]->getMessageWithColumnInfo($code));
29        }
30    }
31
32    public static function provideTestError() {
33        return [
34            ["<?php /*", ["Unterminated comment from 1:7 to 1:9"]],
35            ["<?php /*\n", ["Unterminated comment from 1:7 to 2:1"]],
36            ["<?php \1", ["Unexpected character \"\1\" (ASCII 1) from 1:7 to 1:7"]],
37            ["<?php \0", ["Unexpected null byte from 1:7 to 1:7"]],
38            // Error with potentially emulated token
39            ["<?php ?? \0", ["Unexpected null byte from 1:10 to 1:10"]],
40            ["<?php\n\0\1 foo /* bar", [
41                "Unexpected null byte from 2:1 to 2:1",
42                "Unexpected character \"\1\" (ASCII 1) from 2:2 to 2:2",
43                "Unterminated comment from 2:8 to 2:14"
44            ]],
45        ];
46    }
47
48    public function testDefaultErrorHandler(): void {
49        $this->expectException(Error::class);
50        $this->expectExceptionMessage('Unterminated comment on line 1');
51        $lexer = $this->getLexer();
52        $lexer->tokenize("<?php readonly /*");
53    }
54
55    /**
56     * @dataProvider provideTestLex
57     */
58    public function testLex($code, $expectedTokens): void {
59        $lexer = $this->getLexer();
60        $tokens = $lexer->tokenize($code);
61        foreach ($tokens as $token) {
62            if ($token->id === 0 || $token->isIgnorable()) {
63                continue;
64            }
65
66            $expectedToken = array_shift($expectedTokens);
67
68            $this->assertSame($expectedToken[0], $token->id);
69            $this->assertSame($expectedToken[1], $token->text);
70        }
71    }
72
73    public static function provideTestLex() {
74        return [
75            // tests PHP 8 T_NAME_* emulation
76            [
77                '<?php Foo\Bar \Foo\Bar namespace\Foo\Bar Foo\Bar\\',
78                [
79                    [\T_NAME_QUALIFIED, 'Foo\Bar'],
80                    [\T_NAME_FULLY_QUALIFIED, '\Foo\Bar'],
81                    [\T_NAME_RELATIVE, 'namespace\Foo\Bar'],
82                    [\T_NAME_QUALIFIED, 'Foo\Bar'],
83                    [\T_NS_SEPARATOR, '\\'],
84                ]
85            ],
86            // tests PHP 8 T_NAME_* emulation with reserved keywords
87            [
88                '<?php fn\use \fn\use namespace\fn\use fn\use\\',
89                [
90                    [\T_NAME_QUALIFIED, 'fn\use'],
91                    [\T_NAME_FULLY_QUALIFIED, '\fn\use'],
92                    [\T_NAME_RELATIVE, 'namespace\fn\use'],
93                    [\T_NAME_QUALIFIED, 'fn\use'],
94                    [\T_NS_SEPARATOR, '\\'],
95                ]
96            ],
97        ];
98    }
99
100    public function testGetTokens(): void {
101        $code = '<?php "a";' . "\n" . '// foo' . "\n" . '// bar' . "\n\n" . '"b";';
102        $expectedTokens = [
103            new Token(T_OPEN_TAG, '<?php ', 1, 0),
104            new Token(T_CONSTANT_ENCAPSED_STRING, '"a"', 1, 6),
105            new Token(\ord(';'), ';', 1, 9),
106            new Token(T_WHITESPACE, "\n", 1, 10),
107            new Token(T_COMMENT, '// foo', 2, 11),
108            new Token(T_WHITESPACE, "\n", 2, 17),
109            new Token(T_COMMENT, '// bar', 3, 18),
110            new Token(T_WHITESPACE, "\n\n", 3, 24),
111            new Token(T_CONSTANT_ENCAPSED_STRING, '"b"', 5, 26),
112            new Token(\ord(';'), ';', 5, 29),
113            new Token(0, "\0", 5, 30),
114        ];
115
116        $lexer = $this->getLexer();
117        $this->assertEquals($expectedTokens, $lexer->tokenize($code));
118    }
119}
120