xref: /php-src/ext/tokenizer/tokenizer.c (revision 11accb5c)
1 /*
2    +----------------------------------------------------------------------+
3    | Copyright (c) The PHP Group                                          |
4    +----------------------------------------------------------------------+
5    | This source file is subject to version 3.01 of the PHP license,      |
6    | that is bundled with this package in the file LICENSE, and is        |
7    | available through the world-wide-web at the following url:           |
8    | https://www.php.net/license/3_01.txt                                 |
9    | If you did not receive a copy of the PHP license and are unable to   |
10    | obtain it through the world-wide-web, please send a note to          |
11    | license@php.net so we can mail you a copy immediately.               |
12    +----------------------------------------------------------------------+
13    | Author: Andrei Zmievski <andrei@php.net>                             |
14    +----------------------------------------------------------------------+
15 */
16 
17 #ifdef HAVE_CONFIG_H
18 #include <config.h>
19 #endif
20 
21 #include "php.h"
22 #include "ext/standard/info.h"
23 #include "php_tokenizer.h"
24 
25 #include "zend.h"
26 #include "zend_exceptions.h"
27 #include "zend_language_scanner.h"
28 #include "zend_language_scanner_defs.h"
29 #include <zend_language_parser.h>
30 #include "zend_interfaces.h"
31 
32 #include "tokenizer_data_arginfo.h"
33 #include "tokenizer_arginfo.h"
34 
35 #define zendtext   LANG_SCNG(yy_text)
36 #define zendleng   LANG_SCNG(yy_leng)
37 #define zendcursor LANG_SCNG(yy_cursor)
38 #define zendlimit  LANG_SCNG(yy_limit)
39 
40 static zend_class_entry *php_token_ce;
41 
42 /* {{{ tokenizer_module_entry */
43 zend_module_entry tokenizer_module_entry = {
44 	STANDARD_MODULE_HEADER,
45 	"tokenizer",
46 	ext_functions,
47 	PHP_MINIT(tokenizer),
48 	NULL,
49 	NULL,
50 	NULL,
51 	PHP_MINFO(tokenizer),
52 	PHP_TOKENIZER_VERSION,
53 	STANDARD_MODULE_PROPERTIES
54 };
55 /* }}} */
56 
57 #ifdef COMPILE_DL_TOKENIZER
ZEND_GET_MODULE(tokenizer)58 ZEND_GET_MODULE(tokenizer)
59 #endif
60 
61 static zval *php_token_get_id(zval *obj) {
62 	zval *id = OBJ_PROP_NUM(Z_OBJ_P(obj), 0);
63 	if (Z_ISUNDEF_P(id)) {
64 		zend_throw_error(NULL,
65 			"Typed property PhpToken::$id must not be accessed before initialization");
66 		return NULL;
67 	}
68 
69 	ZVAL_DEREF(id);
70 	ZEND_ASSERT(Z_TYPE_P(id) == IS_LONG);
71 	return id;
72 }
73 
php_token_get_text(zval * obj)74 static zend_string *php_token_get_text(zval *obj) {
75 	zval *text_zval = OBJ_PROP_NUM(Z_OBJ_P(obj), 1);
76 	if (Z_ISUNDEF_P(text_zval)) {
77 		zend_throw_error(NULL,
78 			"Typed property PhpToken::$text must not be accessed before initialization");
79 		return NULL;
80 	}
81 
82 	ZVAL_DEREF(text_zval);
83 	ZEND_ASSERT(Z_TYPE_P(text_zval) == IS_STRING);
84 	return Z_STR_P(text_zval);
85 }
86 
87 static bool tokenize_common(
88 		zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class);
89 
PHP_METHOD(PhpToken,tokenize)90 PHP_METHOD(PhpToken, tokenize)
91 {
92 	zend_string *source;
93 	zend_long flags = 0;
94 	zend_class_entry *token_class;
95 
96 	ZEND_PARSE_PARAMETERS_START(1, 2)
97 		Z_PARAM_STR(source)
98 		Z_PARAM_OPTIONAL
99 		Z_PARAM_LONG(flags)
100 	ZEND_PARSE_PARAMETERS_END();
101 
102 	token_class = zend_get_called_scope(execute_data);
103 
104 	/* Check construction preconditions in advance, so these are not repeated for each token. */
105 	if (token_class->ce_flags & ZEND_ACC_EXPLICIT_ABSTRACT_CLASS) {
106 		zend_throw_error(NULL, "Cannot instantiate abstract class %s", ZSTR_VAL(token_class->name));
107 		RETURN_THROWS();
108 	}
109 	if (zend_update_class_constants(token_class) == FAILURE) {
110 		RETURN_THROWS();
111 	}
112 
113 	if (!tokenize_common(return_value, source, flags, token_class)) {
114 		RETURN_THROWS();
115 	}
116 }
117 
PHP_METHOD(PhpToken,__construct)118 PHP_METHOD(PhpToken, __construct)
119 {
120 	zend_long id;
121 	zend_string *text;
122 	zend_long line = -1;
123 	zend_long pos = -1;
124 	zend_object *obj = Z_OBJ_P(ZEND_THIS);
125 
126 	ZEND_PARSE_PARAMETERS_START(2, 4)
127 		Z_PARAM_LONG(id)
128 		Z_PARAM_STR(text)
129 		Z_PARAM_OPTIONAL
130 		Z_PARAM_LONG(line)
131 		Z_PARAM_LONG(pos)
132 	ZEND_PARSE_PARAMETERS_END();
133 
134 	ZVAL_LONG(OBJ_PROP_NUM(obj, 0), id);
135 	zval_ptr_dtor(OBJ_PROP_NUM(obj, 1));
136 	ZVAL_STR_COPY(OBJ_PROP_NUM(obj, 1), text);
137 	ZVAL_LONG(OBJ_PROP_NUM(obj, 2), line);
138 	ZVAL_LONG(OBJ_PROP_NUM(obj, 3), pos);
139 }
140 
PHP_METHOD(PhpToken,is)141 PHP_METHOD(PhpToken, is)
142 {
143 	zval *kind;
144 
145 	ZEND_PARSE_PARAMETERS_START(1, 1)
146 		Z_PARAM_ZVAL(kind)
147 	ZEND_PARSE_PARAMETERS_END();
148 
149 	if (Z_TYPE_P(kind) == IS_LONG) {
150 		zval *id_zval = php_token_get_id(ZEND_THIS);
151 		if (!id_zval) {
152 			RETURN_THROWS();
153 		}
154 
155 		RETURN_BOOL(Z_LVAL_P(id_zval) == Z_LVAL_P(kind));
156 	} else if (Z_TYPE_P(kind) == IS_STRING) {
157 		zend_string *text = php_token_get_text(ZEND_THIS);
158 		if (!text) {
159 			RETURN_THROWS();
160 		}
161 
162 		RETURN_BOOL(zend_string_equals(text, Z_STR_P(kind)));
163 	} else if (Z_TYPE_P(kind) == IS_ARRAY) {
164 		zval *id_zval = NULL, *entry;
165 		zend_string *text = NULL;
166 		ZEND_HASH_FOREACH_VAL(Z_ARRVAL_P(kind), entry) {
167 			ZVAL_DEREF(entry);
168 			if (Z_TYPE_P(entry) == IS_LONG) {
169 				if (!id_zval) {
170 					id_zval = php_token_get_id(ZEND_THIS);
171 					if (!id_zval) {
172 						RETURN_THROWS();
173 					}
174 				}
175 				if (Z_LVAL_P(id_zval) == Z_LVAL_P(entry)) {
176 					RETURN_TRUE;
177 				}
178 			} else if (Z_TYPE_P(entry) == IS_STRING) {
179 				if (!text) {
180 					text = php_token_get_text(ZEND_THIS);
181 					if (!text) {
182 						RETURN_THROWS();
183 					}
184 				}
185 				if (zend_string_equals(text, Z_STR_P(entry))) {
186 					RETURN_TRUE;
187 				}
188 			} else {
189 				zend_argument_type_error(1, "must only have elements of type string|int, %s given", zend_zval_value_name(entry));
190 				RETURN_THROWS();
191 			}
192 		} ZEND_HASH_FOREACH_END();
193 		RETURN_FALSE;
194 	} else {
195 		zend_argument_type_error(1, "must be of type string|int|array, %s given", zend_zval_value_name(kind));
196 		RETURN_THROWS();
197 	}
198 }
199 
PHP_METHOD(PhpToken,isIgnorable)200 PHP_METHOD(PhpToken, isIgnorable)
201 {
202 	ZEND_PARSE_PARAMETERS_NONE();
203 
204 	zval *id_zval = php_token_get_id(ZEND_THIS);
205 	if (!id_zval) {
206 		RETURN_THROWS();
207 	}
208 
209 	zend_long id = Z_LVAL_P(id_zval);
210 	RETURN_BOOL(id == T_WHITESPACE || id == T_COMMENT || id == T_DOC_COMMENT || id == T_OPEN_TAG);
211 }
212 
PHP_METHOD(PhpToken,getTokenName)213 PHP_METHOD(PhpToken, getTokenName)
214 {
215 	ZEND_PARSE_PARAMETERS_NONE();
216 
217 	zval *id_zval = php_token_get_id(ZEND_THIS);
218 	if (!id_zval) {
219 		RETURN_THROWS();
220 	}
221 
222 	if (Z_LVAL_P(id_zval) < 256) {
223 		RETURN_CHAR(Z_LVAL_P(id_zval));
224 	} else {
225 		const char *token_name = get_token_type_name(Z_LVAL_P(id_zval));
226 		if (!token_name) {
227 			RETURN_NULL();
228 		}
229 
230 		RETURN_STRING(token_name);
231 	}
232 }
233 
PHP_METHOD(PhpToken,__toString)234 PHP_METHOD(PhpToken, __toString)
235 {
236 	ZEND_PARSE_PARAMETERS_NONE();
237 
238 	zend_string *text = php_token_get_text(ZEND_THIS);
239 	if (!text) {
240 		RETURN_THROWS();
241 	}
242 
243 	RETURN_STR_COPY(text);
244 }
245 
246 /* {{{ PHP_MINIT_FUNCTION */
PHP_MINIT_FUNCTION(tokenizer)247 PHP_MINIT_FUNCTION(tokenizer)
248 {
249 	register_tokenizer_data_symbols(module_number);
250 	register_tokenizer_symbols(module_number);
251 	php_token_ce = register_class_PhpToken(zend_ce_stringable);
252 
253 	return SUCCESS;
254 }
255 /* }}} */
256 
257 /* {{{ PHP_MINFO_FUNCTION */
PHP_MINFO_FUNCTION(tokenizer)258 PHP_MINFO_FUNCTION(tokenizer)
259 {
260 	php_info_print_table_start();
261 	php_info_print_table_row(2, "Tokenizer Support", "enabled");
262 	php_info_print_table_end();
263 }
264 /* }}} */
265 
make_str(unsigned char * text,size_t leng,HashTable * interned_strings)266 static zend_string *make_str(unsigned char *text, size_t leng, HashTable *interned_strings) {
267 	if (leng == 1) {
268 		return ZSTR_CHAR(text[0]);
269 	} else if (interned_strings) {
270 		zend_string *interned_str = zend_hash_str_find_ptr(interned_strings, (char *) text, leng);
271 		if (interned_str) {
272 			return zend_string_copy(interned_str);
273 		}
274 		interned_str = zend_string_init((char *) text, leng, 0);
275 		zend_hash_add_new_ptr(interned_strings, interned_str, interned_str);
276 		return interned_str;
277 	} else {
278 		return zend_string_init((char *) text, leng, 0);
279 	}
280 }
281 
add_token(zval * return_value,int token_type,unsigned char * text,size_t leng,int lineno,zend_class_entry * token_class,HashTable * interned_strings)282 static void add_token(
283 		zval *return_value, int token_type, unsigned char *text, size_t leng, int lineno,
284 		zend_class_entry *token_class, HashTable *interned_strings) {
285 	zval token;
286 	if (token_class) {
287 		zend_object *obj = zend_objects_new(token_class);
288 		ZVAL_OBJ(&token, obj);
289 		ZVAL_LONG(OBJ_PROP_NUM(obj, 0), token_type);
290 		ZVAL_STR(OBJ_PROP_NUM(obj, 1), make_str(text, leng, interned_strings));
291 		ZVAL_LONG(OBJ_PROP_NUM(obj, 2), lineno);
292 		ZVAL_LONG(OBJ_PROP_NUM(obj, 3), text - LANG_SCNG(yy_start));
293 
294 		/* If the class is extended with additional properties, initialized them as well. */
295 		if (UNEXPECTED(token_class->default_properties_count > 4)) {
296 			zval *dst = OBJ_PROP_NUM(obj, 4);
297 			zval *src = &token_class->default_properties_table[4];
298 			zval *end = token_class->default_properties_table
299 				+ token_class->default_properties_count;
300 			for (; src < end; src++, dst++) {
301 				ZVAL_COPY_PROP(dst, src);
302 			}
303 		}
304 	} else if (token_type >= 256) {
305 		array_init_size(&token, 3);
306 		zend_hash_real_init_packed(Z_ARRVAL(token));
307 		ZEND_HASH_FILL_PACKED(Z_ARRVAL(token)) {
308 			ZEND_HASH_FILL_SET_LONG(token_type);
309 			ZEND_HASH_FILL_NEXT();
310 			ZEND_HASH_FILL_SET_STR(make_str(text, leng, interned_strings));
311 			ZEND_HASH_FILL_NEXT();
312 			ZEND_HASH_FILL_SET_LONG(lineno);
313 			ZEND_HASH_FILL_NEXT();
314 		} ZEND_HASH_FILL_END();
315 	} else {
316 		ZVAL_STR(&token, make_str(text, leng, interned_strings));
317 	}
318 	zend_hash_next_index_insert_new(Z_ARRVAL_P(return_value), &token);
319 }
320 
tokenize(zval * return_value,zend_string * source,zend_class_entry * token_class)321 static bool tokenize(zval *return_value, zend_string *source, zend_class_entry *token_class)
322 {
323 	zval source_zval;
324 	zend_lex_state original_lex_state;
325 	zval token;
326 	int token_type;
327 	int token_line = 1;
328 	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
329 	HashTable interned_strings;
330 
331 	ZVAL_STR_COPY(&source_zval, source);
332 	zend_save_lexical_state(&original_lex_state);
333 
334 	zend_prepare_string_for_scanning(&source_zval, ZSTR_EMPTY_ALLOC());
335 
336 	LANG_SCNG(yy_state) = yycINITIAL;
337 	zend_hash_init(&interned_strings, 0, NULL, NULL, 0);
338 	array_init(return_value);
339 
340 	while ((token_type = lex_scan(&token, NULL))) {
341 		ZEND_ASSERT(token_type != T_ERROR);
342 
343 		add_token(
344 			return_value, token_type, zendtext, zendleng, token_line,
345 			token_class, &interned_strings);
346 
347 		if (Z_TYPE(token) != IS_UNDEF) {
348 			zval_ptr_dtor_nogc(&token);
349 			ZVAL_UNDEF(&token);
350 		}
351 
352 		/* after T_HALT_COMPILER collect the next three non-dropped tokens */
353 		if (need_tokens != -1) {
354 			if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
355 				&& token_type != T_COMMENT && token_type != T_DOC_COMMENT
356 				&& --need_tokens == 0
357 			) {
358 				/* fetch the rest into a T_INLINE_HTML */
359 				if (zendcursor < zendlimit) {
360 					add_token(
361 						return_value, T_INLINE_HTML, zendcursor, zendlimit - zendcursor,
362 						token_line, token_class, &interned_strings);
363 				}
364 				break;
365 			}
366 		} else if (token_type == T_HALT_COMPILER) {
367 			need_tokens = 3;
368 		}
369 
370 		if (CG(increment_lineno)) {
371 			CG(zend_lineno)++;
372 			CG(increment_lineno) = 0;
373 		}
374 
375 		token_line = CG(zend_lineno);
376 	}
377 
378 	zval_ptr_dtor_str(&source_zval);
379 	zend_restore_lexical_state(&original_lex_state);
380 	zend_hash_destroy(&interned_strings);
381 
382 	return 1;
383 }
384 
385 struct event_context {
386 	zval *tokens;
387 	zend_class_entry *token_class;
388 };
389 
extract_token_id_to_replace(zval * token_zv,const char * text,size_t length)390 static zval *extract_token_id_to_replace(zval *token_zv, const char *text, size_t length) {
391 	zval *id_zv, *text_zv;
392 	ZEND_ASSERT(token_zv);
393 	if (Z_TYPE_P(token_zv) == IS_ARRAY) {
394 		id_zv = zend_hash_index_find(Z_ARRVAL_P(token_zv), 0);
395 		text_zv = zend_hash_index_find(Z_ARRVAL_P(token_zv), 1);
396 	} else if (Z_TYPE_P(token_zv) == IS_OBJECT) {
397 		id_zv = OBJ_PROP_NUM(Z_OBJ_P(token_zv), 0);
398 		text_zv = OBJ_PROP_NUM(Z_OBJ_P(token_zv), 1);
399 	} else {
400 		return NULL;
401 	}
402 
403 	/* There are multiple candidate tokens to which this feedback may apply,
404 	 * check text to make sure this is the right one. */
405 	ZEND_ASSERT(Z_TYPE_P(text_zv) == IS_STRING);
406 	if (Z_STRLEN_P(text_zv) == length && !memcmp(Z_STRVAL_P(text_zv), text, length)) {
407 		return id_zv;
408 	}
409 	return NULL;
410 }
411 
on_event(zend_php_scanner_event event,int token,int line,const char * text,size_t length,void * context)412 static void on_event(
413 		zend_php_scanner_event event, int token, int line,
414 		const char *text, size_t length, void *context)
415 {
416 	struct event_context *ctx = context;
417 
418 	switch (event) {
419 		case ON_TOKEN:
420 			if (token == END) break;
421 			/* Special cases */
422 			if (token == ';' && LANG_SCNG(yy_leng) > 1) { /* ?> or ?>\n or ?>\r\n */
423 				token = T_CLOSE_TAG;
424 			} else if (token == T_ECHO && LANG_SCNG(yy_leng) == sizeof("<?=") - 1) {
425 				token = T_OPEN_TAG_WITH_ECHO;
426 			}
427 			add_token(
428 				ctx->tokens, token, (unsigned char *) text, length, line, ctx->token_class, NULL);
429 			break;
430 		case ON_FEEDBACK: {
431 			HashTable *tokens_ht = Z_ARRVAL_P(ctx->tokens);
432 			zval *token_zv, *id_zv = NULL;
433 			ZEND_HASH_REVERSE_FOREACH_VAL(tokens_ht, token_zv) {
434 				id_zv = extract_token_id_to_replace(token_zv, text, length);
435 				if (id_zv) {
436 					break;
437 				}
438 			} ZEND_HASH_FOREACH_END();
439 			ZEND_ASSERT(id_zv);
440 			ZVAL_LONG(id_zv, token);
441 			break;
442 		}
443 		case ON_STOP:
444 			if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
445 				add_token(ctx->tokens, T_INLINE_HTML, LANG_SCNG(yy_cursor),
446 					LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno),
447 					ctx->token_class, NULL);
448 			}
449 			break;
450 	}
451 }
452 
tokenize_parse(zval * return_value,zend_string * source,zend_class_entry * token_class)453 static bool tokenize_parse(
454 		zval *return_value, zend_string *source, zend_class_entry *token_class)
455 {
456 	zval source_zval;
457 	struct event_context ctx;
458 	zval token_stream;
459 	zend_lex_state original_lex_state;
460 	bool original_in_compilation;
461 	bool success;
462 
463 	ZVAL_STR_COPY(&source_zval, source);
464 
465 	original_in_compilation = CG(in_compilation);
466 	CG(in_compilation) = 1;
467 	zend_save_lexical_state(&original_lex_state);
468 
469 	zend_prepare_string_for_scanning(&source_zval, ZSTR_EMPTY_ALLOC());
470 	array_init(&token_stream);
471 
472 	ctx.tokens = &token_stream;
473 	ctx.token_class = token_class;
474 
475 	CG(ast) = NULL;
476 	CG(ast_arena) = zend_arena_create(1024 * 32);
477 	LANG_SCNG(yy_state) = yycINITIAL;
478 	LANG_SCNG(on_event) = on_event;
479 	LANG_SCNG(on_event_context) = &ctx;
480 
481 	if((success = (zendparse() == SUCCESS))) {
482 		ZVAL_COPY_VALUE(return_value, &token_stream);
483 	} else {
484 		zval_ptr_dtor(&token_stream);
485 	}
486 
487 	zend_ast_destroy(CG(ast));
488 	zend_arena_destroy(CG(ast_arena));
489 
490 	/* restore compiler and scanner global states */
491 	zend_restore_lexical_state(&original_lex_state);
492 	CG(in_compilation) = original_in_compilation;
493 
494 	zval_ptr_dtor_str(&source_zval);
495 
496 	return success;
497 }
498 
tokenize_common(zval * return_value,zend_string * source,zend_long flags,zend_class_entry * token_class)499 static bool tokenize_common(
500 		zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class)
501 {
502 	if (flags & TOKEN_PARSE) {
503 		return tokenize_parse(return_value, source, token_class);
504 	} else {
505 		int success = tokenize(return_value, source, token_class);
506 		/* Normal token_get_all() should not throw. */
507 		zend_clear_exception();
508 		return success;
509 	}
510 }
511 
512 /* }}} */
513 
514 /* {{{ */
PHP_FUNCTION(token_get_all)515 PHP_FUNCTION(token_get_all)
516 {
517 	zend_string *source;
518 	zend_long flags = 0;
519 
520 	ZEND_PARSE_PARAMETERS_START(1, 2)
521 		Z_PARAM_STR(source)
522 		Z_PARAM_OPTIONAL
523 		Z_PARAM_LONG(flags)
524 	ZEND_PARSE_PARAMETERS_END();
525 
526 	if (!tokenize_common(return_value, source, flags, /* token_class */ NULL)) {
527 		RETURN_THROWS();
528 	}
529 }
530 /* }}} */
531 
532 /* {{{ */
PHP_FUNCTION(token_name)533 PHP_FUNCTION(token_name)
534 {
535 	zend_long type;
536 
537 	ZEND_PARSE_PARAMETERS_START(1, 1)
538 		Z_PARAM_LONG(type)
539 	ZEND_PARSE_PARAMETERS_END();
540 
541 	const char *token_name = get_token_type_name(type);
542 	if (!token_name) {
543 		token_name = "UNKNOWN";
544 	}
545 	RETURN_STRING(token_name);
546 }
547 /* }}} */
548