1<?php declare(strict_types=1); 2 3namespace PhpParser\Lexer; 4 5use PhpParser\ErrorHandler; 6use PhpParser\Lexer; 7use PhpParser\LexerTest; 8use PhpParser\Parser\Php7; 9use PhpParser\PhpVersion; 10use PhpParser\Token; 11 12require __DIR__ . '/../../../lib/PhpParser/compatibility_tokens.php'; 13 14class EmulativeTest extends LexerTest { 15 protected function getLexer() { 16 return new Emulative(); 17 } 18 19 /** 20 * @dataProvider provideTestReplaceKeywords 21 */ 22 public function testReplaceKeywords(string $keyword, int $expectedToken): void { 23 $lexer = $this->getLexer(); 24 $code = '<?php ' . $keyword; 25 $this->assertEquals([ 26 new Token(\T_OPEN_TAG, '<?php ', 1, 0), 27 new Token($expectedToken, $keyword, 1, 6), 28 new Token(0, "\0", 1, \strlen($code)), 29 ], $lexer->tokenize($code)); 30 } 31 32 /** 33 * @dataProvider provideTestReplaceKeywords 34 */ 35 public function testReplaceKeywordsUppercase(string $keyword, int $expectedToken): void { 36 $lexer = $this->getLexer(); 37 $code = '<?php ' . strtoupper($keyword); 38 39 $this->assertEquals([ 40 new Token(\T_OPEN_TAG, '<?php ', 1, 0), 41 new Token($expectedToken, \strtoupper($keyword), 1, 6), 42 new Token(0, "\0", 1, \strlen($code)), 43 ], $lexer->tokenize($code)); 44 } 45 46 /** 47 * @dataProvider provideTestReplaceKeywords 48 */ 49 public function testNoReplaceKeywordsAfterObjectOperator(string $keyword): void { 50 $lexer = $this->getLexer(); 51 $code = '<?php ->' . $keyword; 52 53 $this->assertEquals([ 54 new Token(\T_OPEN_TAG, '<?php ', 1, 0), 55 new Token(\T_OBJECT_OPERATOR, '->', 1, 6), 56 new Token(\T_STRING, $keyword, 1, 8), 57 new Token(0, "\0", 1, \strlen($code)), 58 ], $lexer->tokenize($code)); 59 } 60 61 /** 62 * @dataProvider provideTestReplaceKeywords 63 */ 64 public function testNoReplaceKeywordsAfterObjectOperatorWithSpaces(string $keyword): void { 65 $lexer = $this->getLexer(); 66 $code = '<?php -> ' . $keyword; 67 68 $this->assertEquals([ 69 new Token(\T_OPEN_TAG, '<?php ', 1, 0), 70 new Token(\T_OBJECT_OPERATOR, '->', 1, 6), 71 new Token(\T_WHITESPACE, ' ', 1, 8), 72 new Token(\T_STRING, $keyword, 1, 12), 73 new Token(0, "\0", 1, \strlen($code)), 74 ], $lexer->tokenize($code)); 75 } 76 77 /** 78 * @dataProvider provideTestReplaceKeywords 79 */ 80 public function testNoReplaceKeywordsAfterNullsafeObjectOperator(string $keyword): void { 81 $lexer = $this->getLexer(); 82 $code = '<?php ?->' . $keyword; 83 84 $this->assertEquals([ 85 new Token(\T_OPEN_TAG, '<?php ', 1, 0), 86 new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', 1, 6), 87 new Token(\T_STRING, $keyword, 1, 9), 88 new Token(0, "\0", 1, \strlen($code)), 89 ], $lexer->tokenize($code)); 90 } 91 92 public function provideTestReplaceKeywords() { 93 return [ 94 // PHP 8.0 95 ['match', \T_MATCH], 96 97 // PHP 7.4 98 ['fn', \T_FN], 99 100 // PHP 5.5 101 ['finally', \T_FINALLY], 102 ['yield', \T_YIELD], 103 104 // PHP 5.4 105 ['callable', \T_CALLABLE], 106 ['insteadof', \T_INSTEADOF], 107 ['trait', \T_TRAIT], 108 ['__TRAIT__', \T_TRAIT_C], 109 110 // PHP 5.3 111 ['__DIR__', \T_DIR], 112 ['goto', \T_GOTO], 113 ['namespace', \T_NAMESPACE], 114 ['__NAMESPACE__', \T_NS_C], 115 ]; 116 } 117 118 private function assertSameTokens(array $expectedTokens, array $tokens): void { 119 $reducedTokens = []; 120 foreach ($tokens as $token) { 121 if ($token->id === 0 || $token->isIgnorable()) { 122 continue; 123 } 124 $reducedTokens[] = [$token->id, $token->text]; 125 } 126 $this->assertSame($expectedTokens, $reducedTokens); 127 } 128 129 /** 130 * @dataProvider provideTestLexNewFeatures 131 */ 132 public function testLexNewFeatures(string $code, array $expectedTokens): void { 133 $lexer = $this->getLexer(); 134 $this->assertSameTokens($expectedTokens, $lexer->tokenize('<?php ' . $code)); 135 } 136 137 /** 138 * @dataProvider provideTestLexNewFeatures 139 */ 140 public function testLeaveStuffAloneInStrings(string $code): void { 141 $stringifiedToken = '"' . addcslashes($code, '"\\') . '"'; 142 143 $lexer = $this->getLexer(); 144 $fullCode = '<?php ' . $stringifiedToken; 145 146 $this->assertEquals([ 147 new Token(\T_OPEN_TAG, '<?php ', 1, 0), 148 new Token(\T_CONSTANT_ENCAPSED_STRING, $stringifiedToken, 1, 6), 149 new Token(0, "\0", \substr_count($fullCode, "\n") + 1, \strlen($fullCode)), 150 ], $lexer->tokenize($fullCode)); 151 } 152 153 /** 154 * @dataProvider provideTestLexNewFeatures 155 */ 156 public function testErrorAfterEmulation($code): void { 157 $errorHandler = new ErrorHandler\Collecting(); 158 $lexer = $this->getLexer(); 159 $lexer->tokenize('<?php ' . $code . "\0", $errorHandler); 160 161 $errors = $errorHandler->getErrors(); 162 $this->assertCount(1, $errors); 163 164 $error = $errors[0]; 165 $this->assertSame('Unexpected null byte', $error->getRawMessage()); 166 167 $attrs = $error->getAttributes(); 168 $expPos = strlen('<?php ' . $code); 169 $expLine = 1 + substr_count('<?php ' . $code, "\n"); 170 $this->assertSame($expPos, $attrs['startFilePos']); 171 $this->assertSame($expPos, $attrs['endFilePos']); 172 $this->assertSame($expLine, $attrs['startLine']); 173 $this->assertSame($expLine, $attrs['endLine']); 174 } 175 176 public function provideTestLexNewFeatures() { 177 return [ 178 ['yield from', [ 179 [\T_YIELD_FROM, 'yield from'], 180 ]], 181 ["yield\r\nfrom", [ 182 [\T_YIELD_FROM, "yield\r\nfrom"], 183 ]], 184 ['...', [ 185 [\T_ELLIPSIS, '...'], 186 ]], 187 ['**', [ 188 [\T_POW, '**'], 189 ]], 190 ['**=', [ 191 [\T_POW_EQUAL, '**='], 192 ]], 193 ['??', [ 194 [\T_COALESCE, '??'], 195 ]], 196 ['<=>', [ 197 [\T_SPACESHIP, '<=>'], 198 ]], 199 ['0b1010110', [ 200 [\T_LNUMBER, '0b1010110'], 201 ]], 202 ['0b1011010101001010110101010010101011010101010101101011001110111100', [ 203 [\T_DNUMBER, '0b1011010101001010110101010010101011010101010101101011001110111100'], 204 ]], 205 ['\\', [ 206 [\T_NS_SEPARATOR, '\\'], 207 ]], 208 ["<<<'NOWDOC'\nNOWDOC;\n", [ 209 [\T_START_HEREDOC, "<<<'NOWDOC'\n"], 210 [\T_END_HEREDOC, 'NOWDOC'], 211 [ord(';'), ';'], 212 ]], 213 ["<<<'NOWDOC'\nFoobar\nNOWDOC;\n", [ 214 [\T_START_HEREDOC, "<<<'NOWDOC'\n"], 215 [\T_ENCAPSED_AND_WHITESPACE, "Foobar\n"], 216 [\T_END_HEREDOC, 'NOWDOC'], 217 [ord(';'), ';'], 218 ]], 219 220 // PHP 7.3: Flexible heredoc/nowdoc 221 ["<<<LABEL\nLABEL,", [ 222 [\T_START_HEREDOC, "<<<LABEL\n"], 223 [\T_END_HEREDOC, "LABEL"], 224 [ord(','), ','], 225 ]], 226 ["<<<LABEL\n LABEL,", [ 227 [\T_START_HEREDOC, "<<<LABEL\n"], 228 [\T_END_HEREDOC, " LABEL"], 229 [ord(','), ','], 230 ]], 231 ["<<<LABEL\n Foo\n LABEL;", [ 232 [\T_START_HEREDOC, "<<<LABEL\n"], 233 [\T_ENCAPSED_AND_WHITESPACE, " Foo\n"], 234 [\T_END_HEREDOC, " LABEL"], 235 [ord(';'), ';'], 236 ]], 237 ["<<<A\n A,<<<A\n A,", [ 238 [\T_START_HEREDOC, "<<<A\n"], 239 [\T_END_HEREDOC, " A"], 240 [ord(','), ','], 241 [\T_START_HEREDOC, "<<<A\n"], 242 [\T_END_HEREDOC, " A"], 243 [ord(','), ','], 244 ]], 245 ["<<<LABEL\nLABELNOPE\nLABEL\n", [ 246 [\T_START_HEREDOC, "<<<LABEL\n"], 247 [\T_ENCAPSED_AND_WHITESPACE, "LABELNOPE\n"], 248 [\T_END_HEREDOC, "LABEL"], 249 ]], 250 // Interpretation changed 251 ["<<<LABEL\n LABEL\nLABEL\n", [ 252 [\T_START_HEREDOC, "<<<LABEL\n"], 253 [\T_END_HEREDOC, " LABEL"], 254 [\T_STRING, "LABEL"], 255 ]], 256 257 // PHP 7.4: Null coalesce equal 258 ['??=', [ 259 [\T_COALESCE_EQUAL, '??='], 260 ]], 261 262 // PHP 7.4: Number literal separator 263 ['1_000', [ 264 [\T_LNUMBER, '1_000'], 265 ]], 266 ['0x7AFE_F00D', [ 267 [\T_LNUMBER, '0x7AFE_F00D'], 268 ]], 269 ['0b0101_1111', [ 270 [\T_LNUMBER, '0b0101_1111'], 271 ]], 272 ['0137_041', [ 273 [\T_LNUMBER, '0137_041'], 274 ]], 275 ['1_000.0', [ 276 [\T_DNUMBER, '1_000.0'], 277 ]], 278 ['1_0.0', [ 279 [\T_DNUMBER, '1_0.0'] 280 ]], 281 ['1_000_000_000.0', [ 282 [\T_DNUMBER, '1_000_000_000.0'] 283 ]], 284 ['0e1_0', [ 285 [\T_DNUMBER, '0e1_0'] 286 ]], 287 ['1_0e+10', [ 288 [\T_DNUMBER, '1_0e+10'] 289 ]], 290 ['1_0e-10', [ 291 [\T_DNUMBER, '1_0e-10'] 292 ]], 293 ['0b1011010101001010_110101010010_10101101010101_0101101011001_110111100', [ 294 [\T_DNUMBER, '0b1011010101001010_110101010010_10101101010101_0101101011001_110111100'], 295 ]], 296 ['0xFFFF_FFFF_FFFF_FFFF', [ 297 [\T_DNUMBER, '0xFFFF_FFFF_FFFF_FFFF'], 298 ]], 299 ['1_000+1', [ 300 [\T_LNUMBER, '1_000'], 301 [ord('+'), '+'], 302 [\T_LNUMBER, '1'], 303 ]], 304 ['1_0abc', [ 305 [\T_LNUMBER, '1_0'], 306 [\T_STRING, 'abc'], 307 ]], 308 ['?->', [ 309 [\T_NULLSAFE_OBJECT_OPERATOR, '?->'], 310 ]], 311 ['#[Attr]', [ 312 [\T_ATTRIBUTE, '#['], 313 [\T_STRING, 'Attr'], 314 [ord(']'), ']'], 315 ]], 316 ["#[\nAttr\n]", [ 317 [\T_ATTRIBUTE, '#['], 318 [\T_STRING, 'Attr'], 319 [ord(']'), ']'], 320 ]], 321 // Test interaction of two patch-based emulators 322 ["<<<LABEL\n LABEL, #[Attr]", [ 323 [\T_START_HEREDOC, "<<<LABEL\n"], 324 [\T_END_HEREDOC, " LABEL"], 325 [ord(','), ','], 326 [\T_ATTRIBUTE, '#['], 327 [\T_STRING, 'Attr'], 328 [ord(']'), ']'], 329 ]], 330 ["#[Attr] <<<LABEL\n LABEL,", [ 331 [\T_ATTRIBUTE, '#['], 332 [\T_STRING, 'Attr'], 333 [ord(']'), ']'], 334 [\T_START_HEREDOC, "<<<LABEL\n"], 335 [\T_END_HEREDOC, " LABEL"], 336 [ord(','), ','], 337 ]], 338 // Enums use a contextual keyword 339 ['enum Foo {}', [ 340 [\T_ENUM, 'enum'], 341 [\T_STRING, 'Foo'], 342 [ord('{'), '{'], 343 [ord('}'), '}'], 344 ]], 345 ['class Enum {}', [ 346 [\T_CLASS, 'class'], 347 [\T_STRING, 'Enum'], 348 [ord('{'), '{'], 349 [ord('}'), '}'], 350 ]], 351 ['class Enum extends X {}', [ 352 [\T_CLASS, 'class'], 353 [\T_STRING, 'Enum'], 354 [\T_EXTENDS, 'extends'], 355 [\T_STRING, 'X'], 356 [ord('{'), '{'], 357 [ord('}'), '}'], 358 ]], 359 ['class Enum implements X {}', [ 360 [\T_CLASS, 'class'], 361 [\T_STRING, 'Enum'], 362 [\T_IMPLEMENTS, 'implements'], 363 [\T_STRING, 'X'], 364 [ord('{'), '{'], 365 [ord('}'), '}'], 366 ]], 367 ['0o123', [ 368 [\T_LNUMBER, '0o123'], 369 ]], 370 ['0O123', [ 371 [\T_LNUMBER, '0O123'], 372 ]], 373 ['0o1_2_3', [ 374 [\T_LNUMBER, '0o1_2_3'], 375 ]], 376 ['0o1000000000000000000000', [ 377 [\T_DNUMBER, '0o1000000000000000000000'], 378 ]], 379 ['readonly class', [ 380 [\T_READONLY, 'readonly'], 381 [\T_CLASS, 'class'], 382 ]], 383 ['function readonly(', [ 384 [\T_FUNCTION, 'function'], 385 [\T_READONLY, 'readonly'], 386 [ord('('), '('], 387 ]], 388 ['function readonly (', [ 389 [\T_FUNCTION, 'function'], 390 [\T_READONLY, 'readonly'], 391 [ord('('), '('], 392 ]], 393 ]; 394 } 395 396 /** 397 * @dataProvider provideTestTargetVersion 398 */ 399 public function testTargetVersion(string $phpVersion, string $code, array $expectedTokens): void { 400 $lexer = new Emulative(PhpVersion::fromString($phpVersion)); 401 $this->assertSameTokens($expectedTokens, $lexer->tokenize('<?php ' . $code)); 402 } 403 404 public function provideTestTargetVersion() { 405 return [ 406 ['8.0', 'match', [[\T_MATCH, 'match']]], 407 ['7.4', 'match', [[\T_STRING, 'match']]], 408 // Keywords are not case-sensitive. 409 ['8.0', 'MATCH', [[\T_MATCH, 'MATCH']]], 410 ['7.4', 'MATCH', [[\T_STRING, 'MATCH']]], 411 // Tested here to skip testLeaveStuffAloneInStrings. 412 ['8.0', '"$foo?->bar"', [ 413 [ord('"'), '"'], 414 [\T_VARIABLE, '$foo'], 415 [\T_NULLSAFE_OBJECT_OPERATOR, '?->'], 416 [\T_STRING, 'bar'], 417 [ord('"'), '"'], 418 ]], 419 ['8.0', '"$foo?->bar baz"', [ 420 [ord('"'), '"'], 421 [\T_VARIABLE, '$foo'], 422 [\T_NULLSAFE_OBJECT_OPERATOR, '?->'], 423 [\T_STRING, 'bar'], 424 [\T_ENCAPSED_AND_WHITESPACE, ' baz'], 425 [ord('"'), '"'], 426 ]], 427 ]; 428 } 429} 430