1 /*
2 +----------------------------------------------------------------------+
3 | Copyright (c) The PHP Group |
4 +----------------------------------------------------------------------+
5 | This source file is subject to version 3.01 of the PHP license, |
6 | that is bundled with this package in the file LICENSE, and is |
7 | available through the world-wide-web at the following url: |
8 | http://www.php.net/license/3_01.txt |
9 | If you did not receive a copy of the PHP license and are unable to |
10 | obtain it through the world-wide-web, please send a note to |
11 | license@php.net so we can mail you a copy immediately. |
12 +----------------------------------------------------------------------+
13 | Author: Andrei Zmievski <andrei@php.net> |
14 +----------------------------------------------------------------------+
15 */
16
17 #ifdef HAVE_CONFIG_H
18 #include "config.h"
19 #endif
20
21 #include "php.h"
22 #include "php_ini.h"
23 #include "ext/standard/info.h"
24 #include "php_tokenizer.h"
25 #include "tokenizer_arginfo.h"
26
27 #include "zend.h"
28 #include "zend_exceptions.h"
29 #include "zend_language_scanner.h"
30 #include "zend_language_scanner_defs.h"
31 #include <zend_language_parser.h>
32 #include "zend_interfaces.h"
33
34 #define zendtext LANG_SCNG(yy_text)
35 #define zendleng LANG_SCNG(yy_leng)
36 #define zendcursor LANG_SCNG(yy_cursor)
37 #define zendlimit LANG_SCNG(yy_limit)
38
39 #define TOKEN_PARSE (1 << 0)
40
41 zend_class_entry *php_token_ce;
42
tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS)43 void tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS) {
44 REGISTER_LONG_CONSTANT("TOKEN_PARSE", TOKEN_PARSE, CONST_CS|CONST_PERSISTENT);
45 }
46
47 /* {{{ tokenizer_module_entry */
48 zend_module_entry tokenizer_module_entry = {
49 STANDARD_MODULE_HEADER,
50 "tokenizer",
51 ext_functions,
52 PHP_MINIT(tokenizer),
53 NULL,
54 NULL,
55 NULL,
56 PHP_MINFO(tokenizer),
57 PHP_TOKENIZER_VERSION,
58 STANDARD_MODULE_PROPERTIES
59 };
60 /* }}} */
61
62 #ifdef COMPILE_DL_TOKENIZER
ZEND_GET_MODULE(tokenizer)63 ZEND_GET_MODULE(tokenizer)
64 #endif
65
66 static zval *php_token_get_id(zval *obj) {
67 zval *id = OBJ_PROP_NUM(Z_OBJ_P(obj), 0);
68 if (Z_ISUNDEF_P(id)) {
69 zend_throw_error(NULL,
70 "Typed property PhpToken::$id must not be accessed before initialization");
71 return NULL;
72 }
73
74 ZVAL_DEREF(id);
75 ZEND_ASSERT(Z_TYPE_P(id) == IS_LONG);
76 return id;
77 }
78
php_token_get_text(zval * obj)79 static zend_string *php_token_get_text(zval *obj) {
80 zval *text_zval = OBJ_PROP_NUM(Z_OBJ_P(obj), 1);
81 if (Z_ISUNDEF_P(text_zval)) {
82 zend_throw_error(NULL,
83 "Typed property PhpToken::$text must not be accessed before initialization");
84 return NULL;
85 }
86
87 ZVAL_DEREF(text_zval);
88 ZEND_ASSERT(Z_TYPE_P(text_zval) == IS_STRING);
89 return Z_STR_P(text_zval);
90 }
91
92 static zend_bool tokenize_common(
93 zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class);
94
PHP_METHOD(PhpToken,tokenize)95 PHP_METHOD(PhpToken, tokenize)
96 {
97 zend_string *source;
98 zend_long flags = 0;
99 zend_class_entry *token_class;
100
101 ZEND_PARSE_PARAMETERS_START(1, 2)
102 Z_PARAM_STR(source)
103 Z_PARAM_OPTIONAL
104 Z_PARAM_LONG(flags)
105 ZEND_PARSE_PARAMETERS_END();
106
107 token_class = zend_get_called_scope(execute_data);
108
109 /* Check construction preconditions in advance, so these are not repeated for each token. */
110 if (token_class->ce_flags & ZEND_ACC_EXPLICIT_ABSTRACT_CLASS) {
111 zend_throw_error(NULL, "Cannot instantiate abstract class %s", ZSTR_VAL(token_class->name));
112 RETURN_THROWS();
113 }
114 if (zend_update_class_constants(token_class) == FAILURE) {
115 RETURN_THROWS();
116 }
117
118 if (!tokenize_common(return_value, source, flags, token_class)) {
119 RETURN_THROWS();
120 }
121 }
122
PHP_METHOD(PhpToken,__construct)123 PHP_METHOD(PhpToken, __construct)
124 {
125 zend_long id;
126 zend_string *text;
127 zend_long line = -1;
128 zend_long pos = -1;
129 zend_object *obj = Z_OBJ_P(ZEND_THIS);
130
131 ZEND_PARSE_PARAMETERS_START(2, 4)
132 Z_PARAM_LONG(id)
133 Z_PARAM_STR(text)
134 Z_PARAM_OPTIONAL
135 Z_PARAM_LONG(line)
136 Z_PARAM_LONG(pos)
137 ZEND_PARSE_PARAMETERS_END();
138
139 ZVAL_LONG(OBJ_PROP_NUM(obj, 0), id);
140 zval_ptr_dtor(OBJ_PROP_NUM(obj, 1));
141 ZVAL_STR_COPY(OBJ_PROP_NUM(obj, 1), text);
142 ZVAL_LONG(OBJ_PROP_NUM(obj, 2), line);
143 ZVAL_LONG(OBJ_PROP_NUM(obj, 3), pos);
144 }
145
PHP_METHOD(PhpToken,is)146 PHP_METHOD(PhpToken, is)
147 {
148 zval *kind;
149
150 ZEND_PARSE_PARAMETERS_START(1, 1)
151 Z_PARAM_ZVAL(kind)
152 ZEND_PARSE_PARAMETERS_END();
153
154 if (Z_TYPE_P(kind) == IS_LONG) {
155 zval *id_zval = php_token_get_id(ZEND_THIS);
156 if (!id_zval) {
157 RETURN_THROWS();
158 }
159
160 RETURN_BOOL(Z_LVAL_P(id_zval) == Z_LVAL_P(kind));
161 } else if (Z_TYPE_P(kind) == IS_STRING) {
162 zend_string *text = php_token_get_text(ZEND_THIS);
163 if (!text) {
164 RETURN_THROWS();
165 }
166
167 RETURN_BOOL(zend_string_equals(text, Z_STR_P(kind)));
168 } else if (Z_TYPE_P(kind) == IS_ARRAY) {
169 zval *id_zval = NULL, *entry;
170 zend_string *text = NULL;
171 ZEND_HASH_FOREACH_VAL(Z_ARRVAL_P(kind), entry) {
172 ZVAL_DEREF(entry);
173 if (Z_TYPE_P(entry) == IS_LONG) {
174 if (!id_zval) {
175 id_zval = php_token_get_id(ZEND_THIS);
176 if (!id_zval) {
177 RETURN_THROWS();
178 }
179 }
180 if (Z_LVAL_P(id_zval) == Z_LVAL_P(entry)) {
181 RETURN_TRUE;
182 }
183 } else if (Z_TYPE_P(entry) == IS_STRING) {
184 if (!text) {
185 text = php_token_get_text(ZEND_THIS);
186 if (!text) {
187 RETURN_THROWS();
188 }
189 }
190 if (zend_string_equals(text, Z_STR_P(entry))) {
191 RETURN_TRUE;
192 }
193 } else {
194 zend_argument_type_error(1, "must only have elements of type string|int, %s given", zend_zval_type_name(entry));
195 RETURN_THROWS();
196 }
197 } ZEND_HASH_FOREACH_END();
198 RETURN_FALSE;
199 } else {
200 zend_argument_type_error(1, "must be of type string|int|array, %s given", zend_zval_type_name(kind));
201 RETURN_THROWS();
202 }
203 }
204
PHP_METHOD(PhpToken,isIgnorable)205 PHP_METHOD(PhpToken, isIgnorable)
206 {
207 ZEND_PARSE_PARAMETERS_NONE();
208
209 zval *id_zval = php_token_get_id(ZEND_THIS);
210 if (!id_zval) {
211 RETURN_THROWS();
212 }
213
214 zend_long id = Z_LVAL_P(id_zval);
215 RETURN_BOOL(id == T_WHITESPACE || id == T_COMMENT || id == T_DOC_COMMENT || id == T_OPEN_TAG);
216 }
217
PHP_METHOD(PhpToken,getTokenName)218 PHP_METHOD(PhpToken, getTokenName)
219 {
220 ZEND_PARSE_PARAMETERS_NONE();
221
222 zval *id_zval = php_token_get_id(ZEND_THIS);
223 if (!id_zval) {
224 RETURN_THROWS();
225 }
226
227 if (Z_LVAL_P(id_zval) < 256) {
228 RETURN_CHAR(Z_LVAL_P(id_zval));
229 } else {
230 const char *token_name = get_token_type_name(Z_LVAL_P(id_zval));
231 if (!token_name) {
232 RETURN_NULL();
233 }
234
235 RETURN_STRING(token_name);
236 }
237 }
238
PHP_METHOD(PhpToken,__toString)239 PHP_METHOD(PhpToken, __toString)
240 {
241 ZEND_PARSE_PARAMETERS_NONE();
242
243 zend_string *text = php_token_get_text(ZEND_THIS);
244 if (!text) {
245 RETURN_THROWS();
246 }
247
248 RETURN_STR_COPY(text);
249 }
250
251 /* {{{ PHP_MINIT_FUNCTION */
PHP_MINIT_FUNCTION(tokenizer)252 PHP_MINIT_FUNCTION(tokenizer)
253 {
254 zend_class_entry ce;
255 zend_string *name;
256 zval default_val;
257 ZVAL_UNDEF(&default_val);
258
259 tokenizer_register_constants(INIT_FUNC_ARGS_PASSTHRU);
260 tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS_PASSTHRU);
261
262 INIT_CLASS_ENTRY(ce, "PhpToken", class_PhpToken_methods);
263 php_token_ce = zend_register_internal_class(&ce);
264 zend_class_implements(php_token_ce, 1, zend_ce_stringable);
265
266 name = zend_string_init("id", sizeof("id") - 1, 1);
267 zend_declare_typed_property(php_token_ce, name, &default_val, ZEND_ACC_PUBLIC, NULL,
268 (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_LONG));
269 zend_string_release(name);
270
271 name = zend_string_init("text", sizeof("text") - 1, 1);
272 zend_declare_typed_property(php_token_ce, name, &default_val, ZEND_ACC_PUBLIC, NULL,
273 (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_STRING));
274 zend_string_release(name);
275
276 name = zend_string_init("line", sizeof("line") - 1, 1);
277 zend_declare_typed_property(php_token_ce, name, &default_val, ZEND_ACC_PUBLIC, NULL,
278 (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_LONG));
279 zend_string_release(name);
280
281 name = zend_string_init("pos", sizeof("pos") - 1, 1);
282 zend_declare_typed_property(php_token_ce, name, &default_val, ZEND_ACC_PUBLIC, NULL,
283 (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_LONG));
284 zend_string_release(name);
285
286 return SUCCESS;
287 }
288 /* }}} */
289
290 /* {{{ PHP_MINFO_FUNCTION */
PHP_MINFO_FUNCTION(tokenizer)291 PHP_MINFO_FUNCTION(tokenizer)
292 {
293 php_info_print_table_start();
294 php_info_print_table_row(2, "Tokenizer Support", "enabled");
295 php_info_print_table_end();
296 }
297 /* }}} */
298
make_str(unsigned char * text,size_t leng,HashTable * interned_strings)299 static zend_string *make_str(unsigned char *text, size_t leng, HashTable *interned_strings) {
300 if (leng == 1) {
301 return ZSTR_CHAR(text[0]);
302 } else if (interned_strings) {
303 zend_string *interned_str = zend_hash_str_find_ptr(interned_strings, (char *) text, leng);
304 if (interned_str) {
305 return zend_string_copy(interned_str);
306 }
307 interned_str = zend_string_init((char *) text, leng, 0);
308 zend_hash_add_new_ptr(interned_strings, interned_str, interned_str);
309 return interned_str;
310 } else {
311 return zend_string_init((char *) text, leng, 0);
312 }
313 }
314
add_token(zval * return_value,int token_type,unsigned char * text,size_t leng,int lineno,zend_class_entry * token_class,HashTable * interned_strings)315 static void add_token(
316 zval *return_value, int token_type, unsigned char *text, size_t leng, int lineno,
317 zend_class_entry *token_class, HashTable *interned_strings) {
318 zval token;
319 if (token_class) {
320 zend_object *obj = zend_objects_new(token_class);
321 ZVAL_OBJ(&token, obj);
322 ZVAL_LONG(OBJ_PROP_NUM(obj, 0), token_type);
323 ZVAL_STR(OBJ_PROP_NUM(obj, 1), make_str(text, leng, interned_strings));
324 ZVAL_LONG(OBJ_PROP_NUM(obj, 2), lineno);
325 ZVAL_LONG(OBJ_PROP_NUM(obj, 3), text - LANG_SCNG(yy_start));
326
327 /* If the class is extended with additional properties, initialized them as well. */
328 if (UNEXPECTED(token_class->default_properties_count > 4)) {
329 zval *dst = OBJ_PROP_NUM(obj, 4);
330 zval *src = &token_class->default_properties_table[4];
331 zval *end = token_class->default_properties_table
332 + token_class->default_properties_count;
333 for (; src < end; src++, dst++) {
334 ZVAL_COPY_PROP(dst, src);
335 }
336 }
337 } else if (token_type >= 256) {
338 array_init(&token);
339 add_next_index_long(&token, token_type);
340 add_next_index_str(&token, make_str(text, leng, interned_strings));
341 add_next_index_long(&token, lineno);
342 } else {
343 ZVAL_STR(&token, make_str(text, leng, interned_strings));
344 }
345 zend_hash_next_index_insert_new(Z_ARRVAL_P(return_value), &token);
346 }
347
tokenize(zval * return_value,zend_string * source,zend_class_entry * token_class)348 static zend_bool tokenize(zval *return_value, zend_string *source, zend_class_entry *token_class)
349 {
350 zval source_zval;
351 zend_lex_state original_lex_state;
352 zval token;
353 int token_type;
354 int token_line = 1;
355 int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
356 HashTable interned_strings;
357
358 ZVAL_STR_COPY(&source_zval, source);
359 zend_save_lexical_state(&original_lex_state);
360
361 zend_prepare_string_for_scanning(&source_zval, "");
362
363 LANG_SCNG(yy_state) = yycINITIAL;
364 zend_hash_init(&interned_strings, 0, NULL, NULL, 0);
365 array_init(return_value);
366
367 while ((token_type = lex_scan(&token, NULL))) {
368 ZEND_ASSERT(token_type != T_ERROR);
369
370 add_token(
371 return_value, token_type, zendtext, zendleng, token_line,
372 token_class, &interned_strings);
373
374 if (Z_TYPE(token) != IS_UNDEF) {
375 zval_ptr_dtor_nogc(&token);
376 ZVAL_UNDEF(&token);
377 }
378
379 /* after T_HALT_COMPILER collect the next three non-dropped tokens */
380 if (need_tokens != -1) {
381 if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
382 && token_type != T_COMMENT && token_type != T_DOC_COMMENT
383 && --need_tokens == 0
384 ) {
385 /* fetch the rest into a T_INLINE_HTML */
386 if (zendcursor < zendlimit) {
387 add_token(
388 return_value, T_INLINE_HTML, zendcursor, zendlimit - zendcursor,
389 token_line, token_class, &interned_strings);
390 }
391 break;
392 }
393 } else if (token_type == T_HALT_COMPILER) {
394 need_tokens = 3;
395 }
396
397 if (CG(increment_lineno)) {
398 CG(zend_lineno)++;
399 CG(increment_lineno) = 0;
400 }
401
402 token_line = CG(zend_lineno);
403 }
404
405 zval_ptr_dtor_str(&source_zval);
406 zend_restore_lexical_state(&original_lex_state);
407 zend_hash_destroy(&interned_strings);
408
409 return 1;
410 }
411
412 struct event_context {
413 zval *tokens;
414 zend_class_entry *token_class;
415 };
416
extract_token_id_to_replace(zval * token_zv,const char * text,size_t length)417 static zval *extract_token_id_to_replace(zval *token_zv, const char *text, size_t length) {
418 zval *id_zv, *text_zv;
419 ZEND_ASSERT(token_zv);
420 if (Z_TYPE_P(token_zv) == IS_ARRAY) {
421 id_zv = zend_hash_index_find(Z_ARRVAL_P(token_zv), 0);
422 text_zv = zend_hash_index_find(Z_ARRVAL_P(token_zv), 1);
423 } else if (Z_TYPE_P(token_zv) == IS_OBJECT) {
424 id_zv = OBJ_PROP_NUM(Z_OBJ_P(token_zv), 0);
425 text_zv = OBJ_PROP_NUM(Z_OBJ_P(token_zv), 1);
426 } else {
427 return NULL;
428 }
429
430 /* There are multiple candidate tokens to which this feedback may apply,
431 * check text to make sure this is the right one. */
432 ZEND_ASSERT(Z_TYPE_P(text_zv) == IS_STRING);
433 if (Z_STRLEN_P(text_zv) == length && !memcmp(Z_STRVAL_P(text_zv), text, length)) {
434 return id_zv;
435 }
436 return NULL;
437 }
438
on_event(zend_php_scanner_event event,int token,int line,const char * text,size_t length,void * context)439 void on_event(
440 zend_php_scanner_event event, int token, int line,
441 const char *text, size_t length, void *context)
442 {
443 struct event_context *ctx = context;
444
445 switch (event) {
446 case ON_TOKEN:
447 if (token == END) break;
448 /* Special cases */
449 if (token == ';' && LANG_SCNG(yy_leng) > 1) { /* ?> or ?>\n or ?>\r\n */
450 token = T_CLOSE_TAG;
451 } else if (token == T_ECHO && LANG_SCNG(yy_leng) == sizeof("<?=") - 1) {
452 token = T_OPEN_TAG_WITH_ECHO;
453 }
454 add_token(
455 ctx->tokens, token, (unsigned char *) text, length, line, ctx->token_class, NULL);
456 break;
457 case ON_FEEDBACK: {
458 HashTable *tokens_ht = Z_ARRVAL_P(ctx->tokens);
459 zval *token_zv, *id_zv = NULL;
460 ZEND_HASH_REVERSE_FOREACH_VAL(tokens_ht, token_zv) {
461 id_zv = extract_token_id_to_replace(token_zv, text, length);
462 if (id_zv) {
463 break;
464 }
465 } ZEND_HASH_FOREACH_END();
466 ZEND_ASSERT(id_zv);
467 ZVAL_LONG(id_zv, token);
468 break;
469 }
470 case ON_STOP:
471 if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
472 add_token(ctx->tokens, T_INLINE_HTML, LANG_SCNG(yy_cursor),
473 LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno),
474 ctx->token_class, NULL);
475 }
476 break;
477 }
478 }
479
tokenize_parse(zval * return_value,zend_string * source,zend_class_entry * token_class)480 static zend_bool tokenize_parse(
481 zval *return_value, zend_string *source, zend_class_entry *token_class)
482 {
483 zval source_zval;
484 struct event_context ctx;
485 zval token_stream;
486 zend_lex_state original_lex_state;
487 zend_bool original_in_compilation;
488 zend_bool success;
489
490 ZVAL_STR_COPY(&source_zval, source);
491
492 original_in_compilation = CG(in_compilation);
493 CG(in_compilation) = 1;
494 zend_save_lexical_state(&original_lex_state);
495
496 zend_prepare_string_for_scanning(&source_zval, "");
497 array_init(&token_stream);
498
499 ctx.tokens = &token_stream;
500 ctx.token_class = token_class;
501
502 CG(ast) = NULL;
503 CG(ast_arena) = zend_arena_create(1024 * 32);
504 LANG_SCNG(yy_state) = yycINITIAL;
505 LANG_SCNG(on_event) = on_event;
506 LANG_SCNG(on_event_context) = &ctx;
507
508 if((success = (zendparse() == SUCCESS))) {
509 ZVAL_COPY_VALUE(return_value, &token_stream);
510 } else {
511 zval_ptr_dtor(&token_stream);
512 }
513
514 zend_ast_destroy(CG(ast));
515 zend_arena_destroy(CG(ast_arena));
516
517 /* restore compiler and scanner global states */
518 zend_restore_lexical_state(&original_lex_state);
519 CG(in_compilation) = original_in_compilation;
520
521 zval_ptr_dtor_str(&source_zval);
522
523 return success;
524 }
525
tokenize_common(zval * return_value,zend_string * source,zend_long flags,zend_class_entry * token_class)526 static zend_bool tokenize_common(
527 zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class)
528 {
529 if (flags & TOKEN_PARSE) {
530 return tokenize_parse(return_value, source, token_class);
531 } else {
532 int success = tokenize(return_value, source, token_class);
533 /* Normal token_get_all() should not throw. */
534 zend_clear_exception();
535 return success;
536 }
537 }
538
539 /* }}} */
540
541 /* {{{ */
PHP_FUNCTION(token_get_all)542 PHP_FUNCTION(token_get_all)
543 {
544 zend_string *source;
545 zend_long flags = 0;
546
547 ZEND_PARSE_PARAMETERS_START(1, 2)
548 Z_PARAM_STR(source)
549 Z_PARAM_OPTIONAL
550 Z_PARAM_LONG(flags)
551 ZEND_PARSE_PARAMETERS_END();
552
553 if (!tokenize_common(return_value, source, flags, /* token_class */ NULL)) {
554 RETURN_THROWS();
555 }
556 }
557 /* }}} */
558
559 /* {{{ */
PHP_FUNCTION(token_name)560 PHP_FUNCTION(token_name)
561 {
562 zend_long type;
563
564 ZEND_PARSE_PARAMETERS_START(1, 1)
565 Z_PARAM_LONG(type)
566 ZEND_PARSE_PARAMETERS_END();
567
568 const char *token_name = get_token_type_name(type);
569 if (!token_name) {
570 token_name = "UNKNOWN";
571 }
572 RETURN_STRING(token_name);
573 }
574 /* }}} */
575
