1<?php declare(strict_types=1);
2
3namespace PhpParser\Lexer;
4
5use PhpParser\ErrorHandler;
6use PhpParser\Lexer;
7use PhpParser\LexerTest;
8use PhpParser\Parser\Php7;
9use PhpParser\PhpVersion;
10use PhpParser\Token;
11
12require __DIR__ . '/../../../lib/PhpParser/compatibility_tokens.php';
13
14class EmulativeTest extends LexerTest {
15    protected function getLexer() {
16        return new Emulative();
17    }
18
19    /**
20     * @dataProvider provideTestReplaceKeywords
21     */
22    public function testReplaceKeywords(string $keyword, int $expectedToken): void {
23        $lexer = $this->getLexer();
24        $code = '<?php ' . $keyword;
25        $this->assertEquals([
26            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
27            new Token($expectedToken, $keyword, 1, 6),
28            new Token(0, "\0", 1, \strlen($code)),
29        ], $lexer->tokenize($code));
30    }
31
32    /**
33     * @dataProvider provideTestReplaceKeywords
34     */
35    public function testReplaceKeywordsUppercase(string $keyword, int $expectedToken): void {
36        $lexer = $this->getLexer();
37        $code = '<?php ' . strtoupper($keyword);
38
39        $this->assertEquals([
40            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
41            new Token($expectedToken, \strtoupper($keyword), 1, 6),
42            new Token(0, "\0", 1, \strlen($code)),
43        ], $lexer->tokenize($code));
44    }
45
46    /**
47     * @dataProvider provideTestReplaceKeywords
48     */
49    public function testNoReplaceKeywordsAfterObjectOperator(string $keyword): void {
50        $lexer = $this->getLexer();
51        $code = '<?php ->' . $keyword;
52
53        $this->assertEquals([
54            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
55            new Token(\T_OBJECT_OPERATOR, '->', 1, 6),
56            new Token(\T_STRING, $keyword, 1, 8),
57            new Token(0, "\0", 1, \strlen($code)),
58        ], $lexer->tokenize($code));
59    }
60
61    /**
62     * @dataProvider provideTestReplaceKeywords
63     */
64    public function testNoReplaceKeywordsAfterObjectOperatorWithSpaces(string $keyword): void {
65        $lexer = $this->getLexer();
66        $code = '<?php ->    ' . $keyword;
67
68        $this->assertEquals([
69            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
70            new Token(\T_OBJECT_OPERATOR, '->', 1, 6),
71            new Token(\T_WHITESPACE, '    ', 1, 8),
72            new Token(\T_STRING, $keyword, 1, 12),
73            new Token(0, "\0", 1, \strlen($code)),
74        ], $lexer->tokenize($code));
75    }
76
77    /**
78     * @dataProvider provideTestReplaceKeywords
79     */
80    public function testNoReplaceKeywordsAfterNullsafeObjectOperator(string $keyword): void {
81        $lexer = $this->getLexer();
82        $code = '<?php ?->' . $keyword;
83
84        $this->assertEquals([
85            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
86            new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', 1, 6),
87            new Token(\T_STRING, $keyword, 1, 9),
88            new Token(0, "\0", 1, \strlen($code)),
89        ], $lexer->tokenize($code));
90    }
91
92    public function provideTestReplaceKeywords() {
93        return [
94            // PHP 8.0
95            ['match',         \T_MATCH],
96
97            // PHP 7.4
98            ['fn',            \T_FN],
99
100            // PHP 5.5
101            ['finally',       \T_FINALLY],
102            ['yield',         \T_YIELD],
103
104            // PHP 5.4
105            ['callable',      \T_CALLABLE],
106            ['insteadof',     \T_INSTEADOF],
107            ['trait',         \T_TRAIT],
108            ['__TRAIT__',     \T_TRAIT_C],
109
110            // PHP 5.3
111            ['__DIR__',       \T_DIR],
112            ['goto',          \T_GOTO],
113            ['namespace',     \T_NAMESPACE],
114            ['__NAMESPACE__', \T_NS_C],
115        ];
116    }
117
118    private function assertSameTokens(array $expectedTokens, array $tokens): void {
119        $reducedTokens = [];
120        foreach ($tokens as $token) {
121            if ($token->id === 0 || $token->isIgnorable()) {
122                continue;
123            }
124            $reducedTokens[] = [$token->id, $token->text];
125        }
126        $this->assertSame($expectedTokens, $reducedTokens);
127    }
128
129    /**
130     * @dataProvider provideTestLexNewFeatures
131     */
132    public function testLexNewFeatures(string $code, array $expectedTokens): void {
133        $lexer = $this->getLexer();
134        $this->assertSameTokens($expectedTokens, $lexer->tokenize('<?php ' . $code));
135    }
136
137    /**
138     * @dataProvider provideTestLexNewFeatures
139     */
140    public function testLeaveStuffAloneInStrings(string $code): void {
141        $stringifiedToken = '"' . addcslashes($code, '"\\') . '"';
142
143        $lexer = $this->getLexer();
144        $fullCode = '<?php ' . $stringifiedToken;
145
146        $this->assertEquals([
147            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
148            new Token(\T_CONSTANT_ENCAPSED_STRING, $stringifiedToken, 1, 6),
149            new Token(0, "\0", \substr_count($fullCode, "\n") + 1, \strlen($fullCode)),
150        ], $lexer->tokenize($fullCode));
151    }
152
153    /**
154     * @dataProvider provideTestLexNewFeatures
155     */
156    public function testErrorAfterEmulation($code): void {
157        $errorHandler = new ErrorHandler\Collecting();
158        $lexer = $this->getLexer();
159        $lexer->tokenize('<?php ' . $code . "\0", $errorHandler);
160
161        $errors = $errorHandler->getErrors();
162        $this->assertCount(1, $errors);
163
164        $error = $errors[0];
165        $this->assertSame('Unexpected null byte', $error->getRawMessage());
166
167        $attrs = $error->getAttributes();
168        $expPos = strlen('<?php ' . $code);
169        $expLine = 1 + substr_count('<?php ' . $code, "\n");
170        $this->assertSame($expPos, $attrs['startFilePos']);
171        $this->assertSame($expPos, $attrs['endFilePos']);
172        $this->assertSame($expLine, $attrs['startLine']);
173        $this->assertSame($expLine, $attrs['endLine']);
174    }
175
176    public function provideTestLexNewFeatures() {
177        return [
178            ['yield from', [
179                [\T_YIELD_FROM, 'yield from'],
180            ]],
181            ["yield\r\nfrom", [
182                [\T_YIELD_FROM, "yield\r\nfrom"],
183            ]],
184            ['...', [
185                [\T_ELLIPSIS, '...'],
186            ]],
187            ['**', [
188                [\T_POW, '**'],
189            ]],
190            ['**=', [
191                [\T_POW_EQUAL, '**='],
192            ]],
193            ['??', [
194                [\T_COALESCE, '??'],
195            ]],
196            ['<=>', [
197                [\T_SPACESHIP, '<=>'],
198            ]],
199            ['0b1010110', [
200                [\T_LNUMBER, '0b1010110'],
201            ]],
202            ['0b1011010101001010110101010010101011010101010101101011001110111100', [
203                [\T_DNUMBER, '0b1011010101001010110101010010101011010101010101101011001110111100'],
204            ]],
205            ['\\', [
206                [\T_NS_SEPARATOR, '\\'],
207            ]],
208            ["<<<'NOWDOC'\nNOWDOC;\n", [
209                [\T_START_HEREDOC, "<<<'NOWDOC'\n"],
210                [\T_END_HEREDOC, 'NOWDOC'],
211                [ord(';'), ';'],
212            ]],
213            ["<<<'NOWDOC'\nFoobar\nNOWDOC;\n", [
214                [\T_START_HEREDOC, "<<<'NOWDOC'\n"],
215                [\T_ENCAPSED_AND_WHITESPACE, "Foobar\n"],
216                [\T_END_HEREDOC, 'NOWDOC'],
217                [ord(';'), ';'],
218            ]],
219
220            // PHP 7.3: Flexible heredoc/nowdoc
221            ["<<<LABEL\nLABEL,", [
222                [\T_START_HEREDOC, "<<<LABEL\n"],
223                [\T_END_HEREDOC, "LABEL"],
224                [ord(','), ','],
225            ]],
226            ["<<<LABEL\n    LABEL,", [
227                [\T_START_HEREDOC, "<<<LABEL\n"],
228                [\T_END_HEREDOC, "    LABEL"],
229                [ord(','), ','],
230            ]],
231            ["<<<LABEL\n    Foo\n  LABEL;", [
232                [\T_START_HEREDOC, "<<<LABEL\n"],
233                [\T_ENCAPSED_AND_WHITESPACE, "    Foo\n"],
234                [\T_END_HEREDOC, "  LABEL"],
235                [ord(';'), ';'],
236            ]],
237            ["<<<A\n A,<<<A\n A,", [
238                [\T_START_HEREDOC, "<<<A\n"],
239                [\T_END_HEREDOC, " A"],
240                [ord(','), ','],
241                [\T_START_HEREDOC, "<<<A\n"],
242                [\T_END_HEREDOC, " A"],
243                [ord(','), ','],
244            ]],
245            ["<<<LABEL\nLABELNOPE\nLABEL\n", [
246                [\T_START_HEREDOC, "<<<LABEL\n"],
247                [\T_ENCAPSED_AND_WHITESPACE, "LABELNOPE\n"],
248                [\T_END_HEREDOC, "LABEL"],
249            ]],
250            // Interpretation changed
251            ["<<<LABEL\n    LABEL\nLABEL\n", [
252                [\T_START_HEREDOC, "<<<LABEL\n"],
253                [\T_END_HEREDOC, "    LABEL"],
254                [\T_STRING, "LABEL"],
255            ]],
256
257            // PHP 7.4: Null coalesce equal
258            ['??=', [
259                [\T_COALESCE_EQUAL, '??='],
260            ]],
261
262            // PHP 7.4: Number literal separator
263            ['1_000', [
264                [\T_LNUMBER, '1_000'],
265            ]],
266            ['0x7AFE_F00D', [
267                [\T_LNUMBER, '0x7AFE_F00D'],
268            ]],
269            ['0b0101_1111', [
270                [\T_LNUMBER, '0b0101_1111'],
271            ]],
272            ['0137_041', [
273                [\T_LNUMBER, '0137_041'],
274            ]],
275            ['1_000.0', [
276                [\T_DNUMBER, '1_000.0'],
277            ]],
278            ['1_0.0', [
279                [\T_DNUMBER, '1_0.0']
280            ]],
281            ['1_000_000_000.0', [
282                [\T_DNUMBER, '1_000_000_000.0']
283            ]],
284            ['0e1_0', [
285                [\T_DNUMBER, '0e1_0']
286            ]],
287            ['1_0e+10', [
288                [\T_DNUMBER, '1_0e+10']
289            ]],
290            ['1_0e-10', [
291                [\T_DNUMBER, '1_0e-10']
292            ]],
293            ['0b1011010101001010_110101010010_10101101010101_0101101011001_110111100', [
294                [\T_DNUMBER, '0b1011010101001010_110101010010_10101101010101_0101101011001_110111100'],
295            ]],
296            ['0xFFFF_FFFF_FFFF_FFFF', [
297                [\T_DNUMBER, '0xFFFF_FFFF_FFFF_FFFF'],
298            ]],
299            ['1_000+1', [
300                [\T_LNUMBER, '1_000'],
301                [ord('+'), '+'],
302                [\T_LNUMBER, '1'],
303            ]],
304            ['1_0abc', [
305                [\T_LNUMBER, '1_0'],
306                [\T_STRING, 'abc'],
307            ]],
308            ['?->', [
309                [\T_NULLSAFE_OBJECT_OPERATOR, '?->'],
310            ]],
311            ['#[Attr]', [
312                [\T_ATTRIBUTE, '#['],
313                [\T_STRING, 'Attr'],
314                [ord(']'), ']'],
315            ]],
316            ["#[\nAttr\n]", [
317                [\T_ATTRIBUTE, '#['],
318                [\T_STRING, 'Attr'],
319                [ord(']'), ']'],
320            ]],
321            // Test interaction of two patch-based emulators
322            ["<<<LABEL\n    LABEL, #[Attr]", [
323                [\T_START_HEREDOC, "<<<LABEL\n"],
324                [\T_END_HEREDOC, "    LABEL"],
325                [ord(','), ','],
326                [\T_ATTRIBUTE, '#['],
327                [\T_STRING, 'Attr'],
328                [ord(']'), ']'],
329            ]],
330            ["#[Attr] <<<LABEL\n    LABEL,", [
331                [\T_ATTRIBUTE, '#['],
332                [\T_STRING, 'Attr'],
333                [ord(']'), ']'],
334                [\T_START_HEREDOC, "<<<LABEL\n"],
335                [\T_END_HEREDOC, "    LABEL"],
336                [ord(','), ','],
337            ]],
338            // Enums use a contextual keyword
339            ['enum Foo {}', [
340                [\T_ENUM, 'enum'],
341                [\T_STRING, 'Foo'],
342                [ord('{'), '{'],
343                [ord('}'), '}'],
344            ]],
345            ['class Enum {}', [
346                [\T_CLASS, 'class'],
347                [\T_STRING, 'Enum'],
348                [ord('{'), '{'],
349                [ord('}'), '}'],
350            ]],
351            ['class Enum extends X {}', [
352                [\T_CLASS, 'class'],
353                [\T_STRING, 'Enum'],
354                [\T_EXTENDS, 'extends'],
355                [\T_STRING, 'X'],
356                [ord('{'), '{'],
357                [ord('}'), '}'],
358            ]],
359            ['class Enum implements X {}', [
360                [\T_CLASS, 'class'],
361                [\T_STRING, 'Enum'],
362                [\T_IMPLEMENTS, 'implements'],
363                [\T_STRING, 'X'],
364                [ord('{'), '{'],
365                [ord('}'), '}'],
366            ]],
367            ['0o123', [
368                [\T_LNUMBER, '0o123'],
369            ]],
370            ['0O123', [
371                [\T_LNUMBER, '0O123'],
372            ]],
373            ['0o1_2_3', [
374                [\T_LNUMBER, '0o1_2_3'],
375            ]],
376            ['0o1000000000000000000000', [
377                [\T_DNUMBER, '0o1000000000000000000000'],
378            ]],
379            ['readonly class', [
380                [\T_READONLY, 'readonly'],
381                [\T_CLASS, 'class'],
382            ]],
383            ['function readonly(', [
384                [\T_FUNCTION, 'function'],
385                [\T_READONLY, 'readonly'],
386                [ord('('), '('],
387            ]],
388            ['function readonly (', [
389                [\T_FUNCTION, 'function'],
390                [\T_READONLY, 'readonly'],
391                [ord('('), '('],
392            ]],
393        ];
394    }
395
396    /**
397     * @dataProvider provideTestTargetVersion
398     */
399    public function testTargetVersion(string $phpVersion, string $code, array $expectedTokens): void {
400        $lexer = new Emulative(PhpVersion::fromString($phpVersion));
401        $this->assertSameTokens($expectedTokens, $lexer->tokenize('<?php ' . $code));
402    }
403
404    public function provideTestTargetVersion() {
405        return [
406            ['8.0', 'match', [[\T_MATCH, 'match']]],
407            ['7.4', 'match', [[\T_STRING, 'match']]],
408            // Keywords are not case-sensitive.
409            ['8.0', 'MATCH', [[\T_MATCH, 'MATCH']]],
410            ['7.4', 'MATCH', [[\T_STRING, 'MATCH']]],
411            // Tested here to skip testLeaveStuffAloneInStrings.
412            ['8.0', '"$foo?->bar"', [
413                [ord('"'), '"'],
414                [\T_VARIABLE, '$foo'],
415                [\T_NULLSAFE_OBJECT_OPERATOR, '?->'],
416                [\T_STRING, 'bar'],
417                [ord('"'), '"'],
418            ]],
419            ['8.0', '"$foo?->bar baz"', [
420                [ord('"'), '"'],
421                [\T_VARIABLE, '$foo'],
422                [\T_NULLSAFE_OBJECT_OPERATOR, '?->'],
423                [\T_STRING, 'bar'],
424                [\T_ENCAPSED_AND_WHITESPACE, ' baz'],
425                [ord('"'), '"'],
426            ]],
427        ];
428    }
429}
430