Lines Matching refs:tokens
31 $tokens = @Token::tokenize($code);
32 $this->postprocessTokens($tokens, $errorHandler);
38 return $tokens;
69 protected function postprocessTokens(array &$tokens, ErrorHandler $errorHandler): void { argument
76 $numTokens = \count($tokens);
79 $tokens[] = new Token(0, "\0", 1, 0);
84 $token = $tokens[$i];
91 while (isset($tokens[$next]) && $tokens[$next]->id === \T_WHITESPACE) {
94 $followedByVarOrVarArg = isset($tokens[$next]) &&
95 $tokens[$next]->is([\T_VARIABLE, \T_ELLIPSIS]);
103 $lastToken = $tokens[$numTokens - 1];
114 $tokens[] = new Token(0, "\0", $lastToken->getEndLine(), $lastToken->getEndPos());
