1--TEST--
2token_get_all() produces T_BAD_CHARACTER for unexpected characters
3--EXTENSIONS--
4tokenizer
5--FILE--
6<?php
7
8$codes = [
9    "<?php \0 foo",
10    "<?php \1 bar",
11    "<?php \1\2 bar \3",
12];
13
14foreach ($codes as $code) {
15    foreach (token_get_all($code) as $token) {
16        if (is_array($token)) {
17            echo token_name($token[0]), " ", strlen($token[1]), "\n";
18        } else {
19            echo $token, "\n";
20        }
21    }
22    echo "\n";
23}
24
25?>
26--EXPECT--
27T_OPEN_TAG 6
28T_BAD_CHARACTER 1
29T_WHITESPACE 1
30T_STRING 3
31
32T_OPEN_TAG 6
33T_BAD_CHARACTER 1
34T_WHITESPACE 1
35T_STRING 3
36
37T_OPEN_TAG 6
38T_BAD_CHARACTER 1
39T_BAD_CHARACTER 1
40T_WHITESPACE 1
41T_STRING 3
42T_WHITESPACE 1
43T_BAD_CHARACTER 1
44