1--TEST--
2PhpToken::tokenize() method
3--SKIPIF--
4<?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?>
5--FILE--
6<?php
7
8$code = <<<'PHP'
9<?php
10function foo() {
11    echo "bar";
12}
13PHP;
14var_dump(PhpToken::tokenize($code));
15var_dump(PhpToken::tokenize($code, TOKEN_PARSE));
16
17?>
18--EXPECTF--
19array(15) {
20  [0]=>
21  object(PhpToken)#1 (4) {
22    ["id"]=>
23    int(%d)
24    ["text"]=>
25    string(6) "<?php
26"
27    ["line"]=>
28    int(1)
29    ["pos"]=>
30    int(0)
31  }
32  [1]=>
33  object(PhpToken)#2 (4) {
34    ["id"]=>
35    int(%d)
36    ["text"]=>
37    string(8) "function"
38    ["line"]=>
39    int(2)
40    ["pos"]=>
41    int(6)
42  }
43  [2]=>
44  object(PhpToken)#3 (4) {
45    ["id"]=>
46    int(%d)
47    ["text"]=>
48    string(1) " "
49    ["line"]=>
50    int(2)
51    ["pos"]=>
52    int(14)
53  }
54  [3]=>
55  object(PhpToken)#4 (4) {
56    ["id"]=>
57    int(%d)
58    ["text"]=>
59    string(3) "foo"
60    ["line"]=>
61    int(2)
62    ["pos"]=>
63    int(15)
64  }
65  [4]=>
66  object(PhpToken)#5 (4) {
67    ["id"]=>
68    int(%d)
69    ["text"]=>
70    string(1) "("
71    ["line"]=>
72    int(2)
73    ["pos"]=>
74    int(18)
75  }
76  [5]=>
77  object(PhpToken)#6 (4) {
78    ["id"]=>
79    int(%d)
80    ["text"]=>
81    string(1) ")"
82    ["line"]=>
83    int(2)
84    ["pos"]=>
85    int(19)
86  }
87  [6]=>
88  object(PhpToken)#7 (4) {
89    ["id"]=>
90    int(%d)
91    ["text"]=>
92    string(1) " "
93    ["line"]=>
94    int(2)
95    ["pos"]=>
96    int(20)
97  }
98  [7]=>
99  object(PhpToken)#8 (4) {
100    ["id"]=>
101    int(%d)
102    ["text"]=>
103    string(1) "{"
104    ["line"]=>
105    int(2)
106    ["pos"]=>
107    int(21)
108  }
109  [8]=>
110  object(PhpToken)#9 (4) {
111    ["id"]=>
112    int(%d)
113    ["text"]=>
114    string(5) "
115    "
116    ["line"]=>
117    int(2)
118    ["pos"]=>
119    int(22)
120  }
121  [9]=>
122  object(PhpToken)#10 (4) {
123    ["id"]=>
124    int(%d)
125    ["text"]=>
126    string(4) "echo"
127    ["line"]=>
128    int(3)
129    ["pos"]=>
130    int(27)
131  }
132  [10]=>
133  object(PhpToken)#11 (4) {
134    ["id"]=>
135    int(%d)
136    ["text"]=>
137    string(1) " "
138    ["line"]=>
139    int(3)
140    ["pos"]=>
141    int(31)
142  }
143  [11]=>
144  object(PhpToken)#12 (4) {
145    ["id"]=>
146    int(%d)
147    ["text"]=>
148    string(5) ""bar""
149    ["line"]=>
150    int(3)
151    ["pos"]=>
152    int(32)
153  }
154  [12]=>
155  object(PhpToken)#13 (4) {
156    ["id"]=>
157    int(%d)
158    ["text"]=>
159    string(1) ";"
160    ["line"]=>
161    int(3)
162    ["pos"]=>
163    int(37)
164  }
165  [13]=>
166  object(PhpToken)#14 (4) {
167    ["id"]=>
168    int(%d)
169    ["text"]=>
170    string(1) "
171"
172    ["line"]=>
173    int(3)
174    ["pos"]=>
175    int(38)
176  }
177  [14]=>
178  object(PhpToken)#15 (4) {
179    ["id"]=>
180    int(%d)
181    ["text"]=>
182    string(1) "}"
183    ["line"]=>
184    int(4)
185    ["pos"]=>
186    int(39)
187  }
188}
189array(15) {
190  [0]=>
191  object(PhpToken)#15 (4) {
192    ["id"]=>
193    int(%d)
194    ["text"]=>
195    string(6) "<?php
196"
197    ["line"]=>
198    int(1)
199    ["pos"]=>
200    int(0)
201  }
202  [1]=>
203  object(PhpToken)#14 (4) {
204    ["id"]=>
205    int(%d)
206    ["text"]=>
207    string(8) "function"
208    ["line"]=>
209    int(2)
210    ["pos"]=>
211    int(6)
212  }
213  [2]=>
214  object(PhpToken)#13 (4) {
215    ["id"]=>
216    int(%d)
217    ["text"]=>
218    string(1) " "
219    ["line"]=>
220    int(2)
221    ["pos"]=>
222    int(14)
223  }
224  [3]=>
225  object(PhpToken)#12 (4) {
226    ["id"]=>
227    int(%d)
228    ["text"]=>
229    string(3) "foo"
230    ["line"]=>
231    int(2)
232    ["pos"]=>
233    int(15)
234  }
235  [4]=>
236  object(PhpToken)#11 (4) {
237    ["id"]=>
238    int(%d)
239    ["text"]=>
240    string(1) "("
241    ["line"]=>
242    int(2)
243    ["pos"]=>
244    int(18)
245  }
246  [5]=>
247  object(PhpToken)#10 (4) {
248    ["id"]=>
249    int(%d)
250    ["text"]=>
251    string(1) ")"
252    ["line"]=>
253    int(2)
254    ["pos"]=>
255    int(19)
256  }
257  [6]=>
258  object(PhpToken)#9 (4) {
259    ["id"]=>
260    int(%d)
261    ["text"]=>
262    string(1) " "
263    ["line"]=>
264    int(2)
265    ["pos"]=>
266    int(20)
267  }
268  [7]=>
269  object(PhpToken)#8 (4) {
270    ["id"]=>
271    int(%d)
272    ["text"]=>
273    string(1) "{"
274    ["line"]=>
275    int(2)
276    ["pos"]=>
277    int(21)
278  }
279  [8]=>
280  object(PhpToken)#7 (4) {
281    ["id"]=>
282    int(%d)
283    ["text"]=>
284    string(5) "
285    "
286    ["line"]=>
287    int(2)
288    ["pos"]=>
289    int(22)
290  }
291  [9]=>
292  object(PhpToken)#6 (4) {
293    ["id"]=>
294    int(%d)
295    ["text"]=>
296    string(4) "echo"
297    ["line"]=>
298    int(3)
299    ["pos"]=>
300    int(27)
301  }
302  [10]=>
303  object(PhpToken)#5 (4) {
304    ["id"]=>
305    int(%d)
306    ["text"]=>
307    string(1) " "
308    ["line"]=>
309    int(3)
310    ["pos"]=>
311    int(31)
312  }
313  [11]=>
314  object(PhpToken)#4 (4) {
315    ["id"]=>
316    int(%d)
317    ["text"]=>
318    string(5) ""bar""
319    ["line"]=>
320    int(3)
321    ["pos"]=>
322    int(32)
323  }
324  [12]=>
325  object(PhpToken)#3 (4) {
326    ["id"]=>
327    int(%d)
328    ["text"]=>
329    string(1) ";"
330    ["line"]=>
331    int(3)
332    ["pos"]=>
333    int(37)
334  }
335  [13]=>
336  object(PhpToken)#2 (4) {
337    ["id"]=>
338    int(%d)
339    ["text"]=>
340    string(1) "
341"
342    ["line"]=>
343    int(3)
344    ["pos"]=>
345    int(38)
346  }
347  [14]=>
348  object(PhpToken)#1 (4) {
349    ["id"]=>
350    int(%d)
351    ["text"]=>
352    string(1) "}"
353    ["line"]=>
354    int(4)
355    ["pos"]=>
356    int(39)
357  }
358}
359