1<?php declare(strict_types=1);
2
3namespace PhpParser\Lexer;
4
5use PhpParser\ErrorHandler;
6use PhpParser\Lexer;
7use PhpParser\LexerTest;
8use PhpParser\Parser\Php7;
9use PhpParser\PhpVersion;
10use PhpParser\Token;
11
12require __DIR__ . '/../../../lib/PhpParser/compatibility_tokens.php';
13
14class EmulativeTest extends LexerTest {
15    protected function getLexer() {
16        return new Emulative();
17    }
18
19    /**
20     * @dataProvider provideTestReplaceKeywords
21     */
22    public function testReplaceKeywords(string $keyword, int $expectedToken): void {
23        $lexer = $this->getLexer();
24        $code = '<?php ' . $keyword;
25        $this->assertEquals([
26            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
27            new Token($expectedToken, $keyword, 1, 6),
28            new Token(0, "\0", 1, \strlen($code)),
29        ], $lexer->tokenize($code));
30    }
31
32    /**
33     * @dataProvider provideTestReplaceKeywords
34     */
35    public function testReplaceKeywordsUppercase(string $keyword, int $expectedToken): void {
36        $lexer = $this->getLexer();
37        $code = '<?php ' . strtoupper($keyword);
38
39        $this->assertEquals([
40            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
41            new Token($expectedToken, \strtoupper($keyword), 1, 6),
42            new Token(0, "\0", 1, \strlen($code)),
43        ], $lexer->tokenize($code));
44    }
45
46    /**
47     * @dataProvider provideTestReplaceKeywords
48     */
49    public function testNoReplaceKeywordsAfterObjectOperator(string $keyword): void {
50        $lexer = $this->getLexer();
51        $code = '<?php ->' . $keyword;
52
53        $this->assertEquals([
54            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
55            new Token(\T_OBJECT_OPERATOR, '->', 1, 6),
56            new Token(\T_STRING, $keyword, 1, 8),
57            new Token(0, "\0", 1, \strlen($code)),
58        ], $lexer->tokenize($code));
59    }
60
61    /**
62     * @dataProvider provideTestReplaceKeywords
63     */
64    public function testNoReplaceKeywordsAfterObjectOperatorWithSpaces(string $keyword): void {
65        $lexer = $this->getLexer();
66        $code = '<?php ->    ' . $keyword;
67
68        $this->assertEquals([
69            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
70            new Token(\T_OBJECT_OPERATOR, '->', 1, 6),
71            new Token(\T_WHITESPACE, '    ', 1, 8),
72            new Token(\T_STRING, $keyword, 1, 12),
73            new Token(0, "\0", 1, \strlen($code)),
74        ], $lexer->tokenize($code));
75    }
76
77    /**
78     * @dataProvider provideTestReplaceKeywords
79     */
80    public function testNoReplaceKeywordsAfterNullsafeObjectOperator(string $keyword): void {
81        $lexer = $this->getLexer();
82        $code = '<?php ?->' . $keyword;
83
84        $this->assertEquals([
85            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
86            new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', 1, 6),
87            new Token(\T_STRING, $keyword, 1, 9),
88            new Token(0, "\0", 1, \strlen($code)),
89        ], $lexer->tokenize($code));
90    }
91
92    public static function provideTestReplaceKeywords() {
93        return [
94            // PHP 8.4
95            ['__PROPERTY__', \T_PROPERTY_C],
96
97            // PHP 8.0
98            ['match',         \T_MATCH],
99
100            // PHP 7.4
101            ['fn',            \T_FN],
102
103            // PHP 5.5
104            ['finally',       \T_FINALLY],
105            ['yield',         \T_YIELD],
106
107            // PHP 5.4
108            ['callable',      \T_CALLABLE],
109            ['insteadof',     \T_INSTEADOF],
110            ['trait',         \T_TRAIT],
111            ['__TRAIT__',     \T_TRAIT_C],
112
113            // PHP 5.3
114            ['__DIR__',       \T_DIR],
115            ['goto',          \T_GOTO],
116            ['namespace',     \T_NAMESPACE],
117            ['__NAMESPACE__', \T_NS_C],
118        ];
119    }
120
121    private function assertSameTokens(array $expectedTokens, array $tokens): void {
122        $reducedTokens = [];
123        foreach ($tokens as $token) {
124            if ($token->id === 0 || $token->isIgnorable()) {
125                continue;
126            }
127            $reducedTokens[] = [$token->id, $token->text];
128        }
129        $this->assertSame($expectedTokens, $reducedTokens);
130    }
131
132    /**
133     * @dataProvider provideTestLexNewFeatures
134     */
135    public function testLexNewFeatures(string $code, array $expectedTokens): void {
136        $lexer = $this->getLexer();
137        $this->assertSameTokens($expectedTokens, $lexer->tokenize('<?php ' . $code));
138    }
139
140    /**
141     * @dataProvider provideTestLexNewFeatures
142     */
143    public function testLeaveStuffAloneInStrings(string $code): void {
144        $stringifiedToken = '"' . addcslashes($code, '"\\') . '"';
145
146        $lexer = $this->getLexer();
147        $fullCode = '<?php ' . $stringifiedToken;
148
149        $this->assertEquals([
150            new Token(\T_OPEN_TAG, '<?php ', 1, 0),
151            new Token(\T_CONSTANT_ENCAPSED_STRING, $stringifiedToken, 1, 6),
152            new Token(0, "\0", \substr_count($fullCode, "\n") + 1, \strlen($fullCode)),
153        ], $lexer->tokenize($fullCode));
154    }
155
156    /**
157     * @dataProvider provideTestLexNewFeatures
158     */
159    public function testErrorAfterEmulation($code): void {
160        $errorHandler = new ErrorHandler\Collecting();
161        $lexer = $this->getLexer();
162        $lexer->tokenize('<?php ' . $code . "\0", $errorHandler);
163
164        $errors = $errorHandler->getErrors();
165        $this->assertCount(1, $errors);
166
167        $error = $errors[0];
168        $this->assertSame('Unexpected null byte', $error->getRawMessage());
169
170        $attrs = $error->getAttributes();
171        $expPos = strlen('<?php ' . $code);
172        $expLine = 1 + substr_count('<?php ' . $code, "\n");
173        $this->assertSame($expPos, $attrs['startFilePos']);
174        $this->assertSame($expPos, $attrs['endFilePos']);
175        $this->assertSame($expLine, $attrs['startLine']);
176        $this->assertSame($expLine, $attrs['endLine']);
177    }
178
179    public static function provideTestLexNewFeatures() {
180        return [
181            ['yield from', [
182                [\T_YIELD_FROM, 'yield from'],
183            ]],
184            ["yield\r\nfrom", [
185                [\T_YIELD_FROM, "yield\r\nfrom"],
186            ]],
187            ['...', [
188                [\T_ELLIPSIS, '...'],
189            ]],
190            ['**', [
191                [\T_POW, '**'],
192            ]],
193            ['**=', [
194                [\T_POW_EQUAL, '**='],
195            ]],
196            ['??', [
197                [\T_COALESCE, '??'],
198            ]],
199            ['<=>', [
200                [\T_SPACESHIP, '<=>'],
201            ]],
202            ['0b1010110', [
203                [\T_LNUMBER, '0b1010110'],
204            ]],
205            ['0b1011010101001010110101010010101011010101010101101011001110111100', [
206                [\T_DNUMBER, '0b1011010101001010110101010010101011010101010101101011001110111100'],
207            ]],
208            ['\\', [
209                [\T_NS_SEPARATOR, '\\'],
210            ]],
211            ["<<<'NOWDOC'\nNOWDOC;\n", [
212                [\T_START_HEREDOC, "<<<'NOWDOC'\n"],
213                [\T_END_HEREDOC, 'NOWDOC'],
214                [ord(';'), ';'],
215            ]],
216            ["<<<'NOWDOC'\nFoobar\nNOWDOC;\n", [
217                [\T_START_HEREDOC, "<<<'NOWDOC'\n"],
218                [\T_ENCAPSED_AND_WHITESPACE, "Foobar\n"],
219                [\T_END_HEREDOC, 'NOWDOC'],
220                [ord(';'), ';'],
221            ]],
222
223            // PHP 7.3: Flexible heredoc/nowdoc
224            ["<<<LABEL\nLABEL,", [
225                [\T_START_HEREDOC, "<<<LABEL\n"],
226                [\T_END_HEREDOC, "LABEL"],
227                [ord(','), ','],
228            ]],
229            ["<<<LABEL\n    LABEL,", [
230                [\T_START_HEREDOC, "<<<LABEL\n"],
231                [\T_END_HEREDOC, "    LABEL"],
232                [ord(','), ','],
233            ]],
234            ["<<<LABEL\n    Foo\n  LABEL;", [
235                [\T_START_HEREDOC, "<<<LABEL\n"],
236                [\T_ENCAPSED_AND_WHITESPACE, "    Foo\n"],
237                [\T_END_HEREDOC, "  LABEL"],
238                [ord(';'), ';'],
239            ]],
240            ["<<<A\n A,<<<A\n A,", [
241                [\T_START_HEREDOC, "<<<A\n"],
242                [\T_END_HEREDOC, " A"],
243                [ord(','), ','],
244                [\T_START_HEREDOC, "<<<A\n"],
245                [\T_END_HEREDOC, " A"],
246                [ord(','), ','],
247            ]],
248            ["<<<LABEL\nLABELNOPE\nLABEL\n", [
249                [\T_START_HEREDOC, "<<<LABEL\n"],
250                [\T_ENCAPSED_AND_WHITESPACE, "LABELNOPE\n"],
251                [\T_END_HEREDOC, "LABEL"],
252            ]],
253            // Interpretation changed
254            ["<<<LABEL\n    LABEL\nLABEL\n", [
255                [\T_START_HEREDOC, "<<<LABEL\n"],
256                [\T_END_HEREDOC, "    LABEL"],
257                [\T_STRING, "LABEL"],
258            ]],
259
260            // PHP 7.4: Null coalesce equal
261            ['??=', [
262                [\T_COALESCE_EQUAL, '??='],
263            ]],
264
265            // PHP 7.4: Number literal separator
266            ['1_000', [
267                [\T_LNUMBER, '1_000'],
268            ]],
269            ['0x7AFE_F00D', [
270                [\T_LNUMBER, '0x7AFE_F00D'],
271            ]],
272            ['0b0101_1111', [
273                [\T_LNUMBER, '0b0101_1111'],
274            ]],
275            ['0137_041', [
276                [\T_LNUMBER, '0137_041'],
277            ]],
278            ['1_000.0', [
279                [\T_DNUMBER, '1_000.0'],
280            ]],
281            ['1_0.0', [
282                [\T_DNUMBER, '1_0.0']
283            ]],
284            ['1_000_000_000.0', [
285                [\T_DNUMBER, '1_000_000_000.0']
286            ]],
287            ['0e1_0', [
288                [\T_DNUMBER, '0e1_0']
289            ]],
290            ['1_0e+10', [
291                [\T_DNUMBER, '1_0e+10']
292            ]],
293            ['1_0e-10', [
294                [\T_DNUMBER, '1_0e-10']
295            ]],
296            ['0b1011010101001010_110101010010_10101101010101_0101101011001_110111100', [
297                [\T_DNUMBER, '0b1011010101001010_110101010010_10101101010101_0101101011001_110111100'],
298            ]],
299            ['0xFFFF_FFFF_FFFF_FFFF', [
300                [\T_DNUMBER, '0xFFFF_FFFF_FFFF_FFFF'],
301            ]],
302            ['1_000+1', [
303                [\T_LNUMBER, '1_000'],
304                [ord('+'), '+'],
305                [\T_LNUMBER, '1'],
306            ]],
307            ['1_0abc', [
308                [\T_LNUMBER, '1_0'],
309                [\T_STRING, 'abc'],
310            ]],
311            ['?->', [
312                [\T_NULLSAFE_OBJECT_OPERATOR, '?->'],
313            ]],
314            ['#[Attr]', [
315                [\T_ATTRIBUTE, '#['],
316                [\T_STRING, 'Attr'],
317                [ord(']'), ']'],
318            ]],
319            ["#[\nAttr\n]", [
320                [\T_ATTRIBUTE, '#['],
321                [\T_STRING, 'Attr'],
322                [ord(']'), ']'],
323            ]],
324            // Test interaction of two patch-based emulators
325            ["<<<LABEL\n    LABEL, #[Attr]", [
326                [\T_START_HEREDOC, "<<<LABEL\n"],
327                [\T_END_HEREDOC, "    LABEL"],
328                [ord(','), ','],
329                [\T_ATTRIBUTE, '#['],
330                [\T_STRING, 'Attr'],
331                [ord(']'), ']'],
332            ]],
333            ["#[Attr] <<<LABEL\n    LABEL,", [
334                [\T_ATTRIBUTE, '#['],
335                [\T_STRING, 'Attr'],
336                [ord(']'), ']'],
337                [\T_START_HEREDOC, "<<<LABEL\n"],
338                [\T_END_HEREDOC, "    LABEL"],
339                [ord(','), ','],
340            ]],
341            // Enums use a contextual keyword
342            ['enum Foo {}', [
343                [\T_ENUM, 'enum'],
344                [\T_STRING, 'Foo'],
345                [ord('{'), '{'],
346                [ord('}'), '}'],
347            ]],
348            ['class Enum {}', [
349                [\T_CLASS, 'class'],
350                [\T_STRING, 'Enum'],
351                [ord('{'), '{'],
352                [ord('}'), '}'],
353            ]],
354            ['class Enum extends X {}', [
355                [\T_CLASS, 'class'],
356                [\T_STRING, 'Enum'],
357                [\T_EXTENDS, 'extends'],
358                [\T_STRING, 'X'],
359                [ord('{'), '{'],
360                [ord('}'), '}'],
361            ]],
362            ['class Enum implements X {}', [
363                [\T_CLASS, 'class'],
364                [\T_STRING, 'Enum'],
365                [\T_IMPLEMENTS, 'implements'],
366                [\T_STRING, 'X'],
367                [ord('{'), '{'],
368                [ord('}'), '}'],
369            ]],
370            ['0o123', [
371                [\T_LNUMBER, '0o123'],
372            ]],
373            ['0O123', [
374                [\T_LNUMBER, '0O123'],
375            ]],
376            ['0o1_2_3', [
377                [\T_LNUMBER, '0o1_2_3'],
378            ]],
379            ['0o1000000000000000000000', [
380                [\T_DNUMBER, '0o1000000000000000000000'],
381            ]],
382            ['readonly class', [
383                [\T_READONLY, 'readonly'],
384                [\T_CLASS, 'class'],
385            ]],
386            ['function readonly(', [
387                [\T_FUNCTION, 'function'],
388                [\T_READONLY, 'readonly'],
389                [ord('('), '('],
390            ]],
391            ['function readonly (', [
392                [\T_FUNCTION, 'function'],
393                [\T_READONLY, 'readonly'],
394                [ord('('), '('],
395            ]],
396
397            // PHP 8.4: Asymmetric visibility modifiers
398            ['private(set)', [
399                [\T_PRIVATE_SET, 'private(set)']
400            ]],
401            ['PROTECTED(SET)', [
402                [\T_PROTECTED_SET, 'PROTECTED(SET)']
403            ]],
404            ['Public(Set)', [
405                [\T_PUBLIC_SET, 'Public(Set)']
406            ]],
407            ['public (set)', [
408                [\T_PUBLIC, 'public'],
409                [\ord('('), '('],
410                [\T_STRING, 'set'],
411                [\ord(')'), ')'],
412            ]],
413            ['->public(set)', [
414                [\T_OBJECT_OPERATOR, '->'],
415                [\T_STRING, 'public'],
416                [\ord('('), '('],
417                [\T_STRING, 'set'],
418                [\ord(')'), ')'],
419            ]],
420            ['?-> public(set)', [
421                [\T_NULLSAFE_OBJECT_OPERATOR, '?->'],
422                [\T_STRING, 'public'],
423                [\ord('('), '('],
424                [\T_STRING, 'set'],
425                [\ord(')'), ')'],
426            ]],
427        ];
428    }
429
430    /**
431     * @dataProvider provideTestTargetVersion
432     */
433    public function testTargetVersion(string $phpVersion, string $code, array $expectedTokens): void {
434        $lexer = new Emulative(PhpVersion::fromString($phpVersion));
435        $this->assertSameTokens($expectedTokens, $lexer->tokenize('<?php ' . $code));
436    }
437
438    public static function provideTestTargetVersion() {
439        return [
440            ['8.0', 'match', [[\T_MATCH, 'match']]],
441            ['7.4', 'match', [[\T_STRING, 'match']]],
442            // Keywords are not case-sensitive.
443            ['8.0', 'MATCH', [[\T_MATCH, 'MATCH']]],
444            ['7.4', 'MATCH', [[\T_STRING, 'MATCH']]],
445            // Tested here to skip testLeaveStuffAloneInStrings.
446            ['8.0', '"$foo?->bar"', [
447                [ord('"'), '"'],
448                [\T_VARIABLE, '$foo'],
449                [\T_NULLSAFE_OBJECT_OPERATOR, '?->'],
450                [\T_STRING, 'bar'],
451                [ord('"'), '"'],
452            ]],
453            ['8.0', '"$foo?->bar baz"', [
454                [ord('"'), '"'],
455                [\T_VARIABLE, '$foo'],
456                [\T_NULLSAFE_OBJECT_OPERATOR, '?->'],
457                [\T_STRING, 'bar'],
458                [\T_ENCAPSED_AND_WHITESPACE, ' baz'],
459                [ord('"'), '"'],
460            ]],
461            ['8.4', '__PROPERTY__', [[\T_PROPERTY_C, '__PROPERTY__']]],
462            ['8.3', '__PROPERTY__', [[\T_STRING, '__PROPERTY__']]],
463            ['8.4', '__property__', [[\T_PROPERTY_C, '__property__']]],
464            ['8.3', '__property__', [[\T_STRING, '__property__']]],
465            ['8.4', 'public(set)', [
466                [\T_PUBLIC_SET, 'public(set)'],
467            ]],
468            ['8.3', 'public(set)', [
469                [\T_PUBLIC, 'public'],
470                [\ord('('), '('],
471                [\T_STRING, 'set'],
472                [\ord(')'), ')']
473            ]],
474        ];
475    }
476}
477