postprocessTokens($tokens, $errorHandler); if (false !== $scream) { ini_set('xdebug.scream', $scream); } return $tokens; } private function handleInvalidCharacter(Token $token, ErrorHandler $errorHandler): void { $chr = $token->text; if ($chr === "\0") { // PHP cuts error message after null byte, so need special case $errorMsg = 'Unexpected null byte'; } else { $errorMsg = sprintf( 'Unexpected character "%s" (ASCII %d)', $chr, ord($chr) ); } $errorHandler->handleError(new Error($errorMsg, [ 'startLine' => $token->line, 'endLine' => $token->line, 'startFilePos' => $token->pos, 'endFilePos' => $token->pos, ])); } private function isUnterminatedComment(Token $token): bool { return $token->is([\T_COMMENT, \T_DOC_COMMENT]) && substr($token->text, 0, 2) === '/*' && substr($token->text, -2) !== '*/'; } /** * @param list $tokens */ protected function postprocessTokens(array &$tokens, ErrorHandler $errorHandler): void { // This function reports errors (bad characters and unterminated comments) in the token // array, and performs certain canonicalizations: // * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and // T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types. // * Add a sentinel token with ID 0. $numTokens = \count($tokens); if ($numTokens === 0) { // Empty input edge case: Just add the sentinel token. $tokens[] = new Token(0, "\0", 1, 0); return; } for ($i = 0; $i < $numTokens; $i++) { $token = $tokens[$i]; if ($token->id === \T_BAD_CHARACTER) { $this->handleInvalidCharacter($token, $errorHandler); } if ($token->id === \ord('&')) { $next = $i + 1; while (isset($tokens[$next]) && $tokens[$next]->id === \T_WHITESPACE) { $next++; } $followedByVarOrVarArg = isset($tokens[$next]) && $tokens[$next]->is([\T_VARIABLE, \T_ELLIPSIS]); $token->id = $followedByVarOrVarArg ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG; } } // Check for unterminated comment $lastToken = $tokens[$numTokens - 1]; if ($this->isUnterminatedComment($lastToken)) { $errorHandler->handleError(new Error('Unterminated comment', [ 'startLine' => $lastToken->line, 'endLine' => $lastToken->getEndLine(), 'startFilePos' => $lastToken->pos, 'endFilePos' => $lastToken->getEndPos(), ])); } // Add sentinel token. $tokens[] = new Token(0, "\0", $lastToken->getEndLine(), $lastToken->getEndPos()); } }