xref: /PHP-8.3/ext/tokenizer/tokenizer.c (revision 7936c808)
1 /*
2    +----------------------------------------------------------------------+
3    | Copyright (c) The PHP Group                                          |
4    +----------------------------------------------------------------------+
5    | This source file is subject to version 3.01 of the PHP license,      |
6    | that is bundled with this package in the file LICENSE, and is        |
7    | available through the world-wide-web at the following url:           |
8    | https://www.php.net/license/3_01.txt                                 |
9    | If you did not receive a copy of the PHP license and are unable to   |
10    | obtain it through the world-wide-web, please send a note to          |
11    | license@php.net so we can mail you a copy immediately.               |
12    +----------------------------------------------------------------------+
13    | Author: Andrei Zmievski <andrei@php.net>                             |
14    +----------------------------------------------------------------------+
15 */
16 
17 #ifdef HAVE_CONFIG_H
18 #include "config.h"
19 #endif
20 
21 #include "php.h"
22 #include "php_ini.h"
23 #include "ext/standard/info.h"
24 #include "php_tokenizer.h"
25 
26 #include "zend.h"
27 #include "zend_exceptions.h"
28 #include "zend_language_scanner.h"
29 #include "zend_language_scanner_defs.h"
30 #include <zend_language_parser.h>
31 #include "zend_interfaces.h"
32 
33 #include "tokenizer_data_arginfo.h"
34 #include "tokenizer_arginfo.h"
35 
36 #define zendtext   LANG_SCNG(yy_text)
37 #define zendleng   LANG_SCNG(yy_leng)
38 #define zendcursor LANG_SCNG(yy_cursor)
39 #define zendlimit  LANG_SCNG(yy_limit)
40 
41 zend_class_entry *php_token_ce;
42 
43 /* {{{ tokenizer_module_entry */
44 zend_module_entry tokenizer_module_entry = {
45 	STANDARD_MODULE_HEADER,
46 	"tokenizer",
47 	ext_functions,
48 	PHP_MINIT(tokenizer),
49 	NULL,
50 	NULL,
51 	NULL,
52 	PHP_MINFO(tokenizer),
53 	PHP_TOKENIZER_VERSION,
54 	STANDARD_MODULE_PROPERTIES
55 };
56 /* }}} */
57 
58 #ifdef COMPILE_DL_TOKENIZER
ZEND_GET_MODULE(tokenizer)59 ZEND_GET_MODULE(tokenizer)
60 #endif
61 
62 static zval *php_token_get_id(zval *obj) {
63 	zval *id = OBJ_PROP_NUM(Z_OBJ_P(obj), 0);
64 	if (Z_ISUNDEF_P(id)) {
65 		zend_throw_error(NULL,
66 			"Typed property PhpToken::$id must not be accessed before initialization");
67 		return NULL;
68 	}
69 
70 	ZVAL_DEREF(id);
71 	ZEND_ASSERT(Z_TYPE_P(id) == IS_LONG);
72 	return id;
73 }
74 
php_token_get_text(zval * obj)75 static zend_string *php_token_get_text(zval *obj) {
76 	zval *text_zval = OBJ_PROP_NUM(Z_OBJ_P(obj), 1);
77 	if (Z_ISUNDEF_P(text_zval)) {
78 		zend_throw_error(NULL,
79 			"Typed property PhpToken::$text must not be accessed before initialization");
80 		return NULL;
81 	}
82 
83 	ZVAL_DEREF(text_zval);
84 	ZEND_ASSERT(Z_TYPE_P(text_zval) == IS_STRING);
85 	return Z_STR_P(text_zval);
86 }
87 
88 static bool tokenize_common(
89 		zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class);
90 
PHP_METHOD(PhpToken,tokenize)91 PHP_METHOD(PhpToken, tokenize)
92 {
93 	zend_string *source;
94 	zend_long flags = 0;
95 	zend_class_entry *token_class;
96 
97 	ZEND_PARSE_PARAMETERS_START(1, 2)
98 		Z_PARAM_STR(source)
99 		Z_PARAM_OPTIONAL
100 		Z_PARAM_LONG(flags)
101 	ZEND_PARSE_PARAMETERS_END();
102 
103 	token_class = zend_get_called_scope(execute_data);
104 
105 	/* Check construction preconditions in advance, so these are not repeated for each token. */
106 	if (token_class->ce_flags & ZEND_ACC_EXPLICIT_ABSTRACT_CLASS) {
107 		zend_throw_error(NULL, "Cannot instantiate abstract class %s", ZSTR_VAL(token_class->name));
108 		RETURN_THROWS();
109 	}
110 	if (zend_update_class_constants(token_class) == FAILURE) {
111 		RETURN_THROWS();
112 	}
113 
114 	if (!tokenize_common(return_value, source, flags, token_class)) {
115 		RETURN_THROWS();
116 	}
117 }
118 
PHP_METHOD(PhpToken,__construct)119 PHP_METHOD(PhpToken, __construct)
120 {
121 	zend_long id;
122 	zend_string *text;
123 	zend_long line = -1;
124 	zend_long pos = -1;
125 	zend_object *obj = Z_OBJ_P(ZEND_THIS);
126 
127 	ZEND_PARSE_PARAMETERS_START(2, 4)
128 		Z_PARAM_LONG(id)
129 		Z_PARAM_STR(text)
130 		Z_PARAM_OPTIONAL
131 		Z_PARAM_LONG(line)
132 		Z_PARAM_LONG(pos)
133 	ZEND_PARSE_PARAMETERS_END();
134 
135 	ZVAL_LONG(OBJ_PROP_NUM(obj, 0), id);
136 	zval_ptr_dtor(OBJ_PROP_NUM(obj, 1));
137 	ZVAL_STR_COPY(OBJ_PROP_NUM(obj, 1), text);
138 	ZVAL_LONG(OBJ_PROP_NUM(obj, 2), line);
139 	ZVAL_LONG(OBJ_PROP_NUM(obj, 3), pos);
140 }
141 
PHP_METHOD(PhpToken,is)142 PHP_METHOD(PhpToken, is)
143 {
144 	zval *kind;
145 
146 	ZEND_PARSE_PARAMETERS_START(1, 1)
147 		Z_PARAM_ZVAL(kind)
148 	ZEND_PARSE_PARAMETERS_END();
149 
150 	if (Z_TYPE_P(kind) == IS_LONG) {
151 		zval *id_zval = php_token_get_id(ZEND_THIS);
152 		if (!id_zval) {
153 			RETURN_THROWS();
154 		}
155 
156 		RETURN_BOOL(Z_LVAL_P(id_zval) == Z_LVAL_P(kind));
157 	} else if (Z_TYPE_P(kind) == IS_STRING) {
158 		zend_string *text = php_token_get_text(ZEND_THIS);
159 		if (!text) {
160 			RETURN_THROWS();
161 		}
162 
163 		RETURN_BOOL(zend_string_equals(text, Z_STR_P(kind)));
164 	} else if (Z_TYPE_P(kind) == IS_ARRAY) {
165 		zval *id_zval = NULL, *entry;
166 		zend_string *text = NULL;
167 		ZEND_HASH_FOREACH_VAL(Z_ARRVAL_P(kind), entry) {
168 			ZVAL_DEREF(entry);
169 			if (Z_TYPE_P(entry) == IS_LONG) {
170 				if (!id_zval) {
171 					id_zval = php_token_get_id(ZEND_THIS);
172 					if (!id_zval) {
173 						RETURN_THROWS();
174 					}
175 				}
176 				if (Z_LVAL_P(id_zval) == Z_LVAL_P(entry)) {
177 					RETURN_TRUE;
178 				}
179 			} else if (Z_TYPE_P(entry) == IS_STRING) {
180 				if (!text) {
181 					text = php_token_get_text(ZEND_THIS);
182 					if (!text) {
183 						RETURN_THROWS();
184 					}
185 				}
186 				if (zend_string_equals(text, Z_STR_P(entry))) {
187 					RETURN_TRUE;
188 				}
189 			} else {
190 				zend_argument_type_error(1, "must only have elements of type string|int, %s given", zend_zval_value_name(entry));
191 				RETURN_THROWS();
192 			}
193 		} ZEND_HASH_FOREACH_END();
194 		RETURN_FALSE;
195 	} else {
196 		zend_argument_type_error(1, "must be of type string|int|array, %s given", zend_zval_value_name(kind));
197 		RETURN_THROWS();
198 	}
199 }
200 
PHP_METHOD(PhpToken,isIgnorable)201 PHP_METHOD(PhpToken, isIgnorable)
202 {
203 	ZEND_PARSE_PARAMETERS_NONE();
204 
205 	zval *id_zval = php_token_get_id(ZEND_THIS);
206 	if (!id_zval) {
207 		RETURN_THROWS();
208 	}
209 
210 	zend_long id = Z_LVAL_P(id_zval);
211 	RETURN_BOOL(id == T_WHITESPACE || id == T_COMMENT || id == T_DOC_COMMENT || id == T_OPEN_TAG);
212 }
213 
PHP_METHOD(PhpToken,getTokenName)214 PHP_METHOD(PhpToken, getTokenName)
215 {
216 	ZEND_PARSE_PARAMETERS_NONE();
217 
218 	zval *id_zval = php_token_get_id(ZEND_THIS);
219 	if (!id_zval) {
220 		RETURN_THROWS();
221 	}
222 
223 	if (Z_LVAL_P(id_zval) < 256) {
224 		RETURN_CHAR(Z_LVAL_P(id_zval));
225 	} else {
226 		const char *token_name = get_token_type_name(Z_LVAL_P(id_zval));
227 		if (!token_name) {
228 			RETURN_NULL();
229 		}
230 
231 		RETURN_STRING(token_name);
232 	}
233 }
234 
PHP_METHOD(PhpToken,__toString)235 PHP_METHOD(PhpToken, __toString)
236 {
237 	ZEND_PARSE_PARAMETERS_NONE();
238 
239 	zend_string *text = php_token_get_text(ZEND_THIS);
240 	if (!text) {
241 		RETURN_THROWS();
242 	}
243 
244 	RETURN_STR_COPY(text);
245 }
246 
247 /* {{{ PHP_MINIT_FUNCTION */
PHP_MINIT_FUNCTION(tokenizer)248 PHP_MINIT_FUNCTION(tokenizer)
249 {
250 	register_tokenizer_data_symbols(module_number);
251 	register_tokenizer_symbols(module_number);
252 	php_token_ce = register_class_PhpToken(zend_ce_stringable);
253 
254 	return SUCCESS;
255 }
256 /* }}} */
257 
258 /* {{{ PHP_MINFO_FUNCTION */
PHP_MINFO_FUNCTION(tokenizer)259 PHP_MINFO_FUNCTION(tokenizer)
260 {
261 	php_info_print_table_start();
262 	php_info_print_table_row(2, "Tokenizer Support", "enabled");
263 	php_info_print_table_end();
264 }
265 /* }}} */
266 
make_str(unsigned char * text,size_t leng,HashTable * interned_strings)267 static zend_string *make_str(unsigned char *text, size_t leng, HashTable *interned_strings) {
268 	if (leng == 1) {
269 		return ZSTR_CHAR(text[0]);
270 	} else if (interned_strings) {
271 		zend_string *interned_str = zend_hash_str_find_ptr(interned_strings, (char *) text, leng);
272 		if (interned_str) {
273 			return zend_string_copy(interned_str);
274 		}
275 		interned_str = zend_string_init((char *) text, leng, 0);
276 		zend_hash_add_new_ptr(interned_strings, interned_str, interned_str);
277 		return interned_str;
278 	} else {
279 		return zend_string_init((char *) text, leng, 0);
280 	}
281 }
282 
add_token(zval * return_value,int token_type,unsigned char * text,size_t leng,int lineno,zend_class_entry * token_class,HashTable * interned_strings)283 static void add_token(
284 		zval *return_value, int token_type, unsigned char *text, size_t leng, int lineno,
285 		zend_class_entry *token_class, HashTable *interned_strings) {
286 	zval token;
287 	if (token_class) {
288 		zend_object *obj = zend_objects_new(token_class);
289 		ZVAL_OBJ(&token, obj);
290 		ZVAL_LONG(OBJ_PROP_NUM(obj, 0), token_type);
291 		ZVAL_STR(OBJ_PROP_NUM(obj, 1), make_str(text, leng, interned_strings));
292 		ZVAL_LONG(OBJ_PROP_NUM(obj, 2), lineno);
293 		ZVAL_LONG(OBJ_PROP_NUM(obj, 3), text - LANG_SCNG(yy_start));
294 
295 		/* If the class is extended with additional properties, initialized them as well. */
296 		if (UNEXPECTED(token_class->default_properties_count > 4)) {
297 			zval *dst = OBJ_PROP_NUM(obj, 4);
298 			zval *src = &token_class->default_properties_table[4];
299 			zval *end = token_class->default_properties_table
300 				+ token_class->default_properties_count;
301 			for (; src < end; src++, dst++) {
302 				ZVAL_COPY_PROP(dst, src);
303 			}
304 		}
305 	} else if (token_type >= 256) {
306 		array_init_size(&token, 3);
307 		zend_hash_real_init_packed(Z_ARRVAL(token));
308 		ZEND_HASH_FILL_PACKED(Z_ARRVAL(token)) {
309 			ZEND_HASH_FILL_SET_LONG(token_type);
310 			ZEND_HASH_FILL_NEXT();
311 			ZEND_HASH_FILL_SET_STR(make_str(text, leng, interned_strings));
312 			ZEND_HASH_FILL_NEXT();
313 			ZEND_HASH_FILL_SET_LONG(lineno);
314 			ZEND_HASH_FILL_NEXT();
315 		} ZEND_HASH_FILL_END();
316 	} else {
317 		ZVAL_STR(&token, make_str(text, leng, interned_strings));
318 	}
319 	zend_hash_next_index_insert_new(Z_ARRVAL_P(return_value), &token);
320 }
321 
tokenize(zval * return_value,zend_string * source,zend_class_entry * token_class)322 static bool tokenize(zval *return_value, zend_string *source, zend_class_entry *token_class)
323 {
324 	zval source_zval;
325 	zend_lex_state original_lex_state;
326 	zval token;
327 	int token_type;
328 	int token_line = 1;
329 	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
330 	HashTable interned_strings;
331 
332 	ZVAL_STR_COPY(&source_zval, source);
333 	zend_save_lexical_state(&original_lex_state);
334 
335 	zend_prepare_string_for_scanning(&source_zval, ZSTR_EMPTY_ALLOC());
336 
337 	LANG_SCNG(yy_state) = yycINITIAL;
338 	zend_hash_init(&interned_strings, 0, NULL, NULL, 0);
339 	array_init(return_value);
340 
341 	while ((token_type = lex_scan(&token, NULL))) {
342 		ZEND_ASSERT(token_type != T_ERROR);
343 
344 		add_token(
345 			return_value, token_type, zendtext, zendleng, token_line,
346 			token_class, &interned_strings);
347 
348 		if (Z_TYPE(token) != IS_UNDEF) {
349 			zval_ptr_dtor_nogc(&token);
350 			ZVAL_UNDEF(&token);
351 		}
352 
353 		/* after T_HALT_COMPILER collect the next three non-dropped tokens */
354 		if (need_tokens != -1) {
355 			if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
356 				&& token_type != T_COMMENT && token_type != T_DOC_COMMENT
357 				&& --need_tokens == 0
358 			) {
359 				/* fetch the rest into a T_INLINE_HTML */
360 				if (zendcursor < zendlimit) {
361 					add_token(
362 						return_value, T_INLINE_HTML, zendcursor, zendlimit - zendcursor,
363 						token_line, token_class, &interned_strings);
364 				}
365 				break;
366 			}
367 		} else if (token_type == T_HALT_COMPILER) {
368 			need_tokens = 3;
369 		}
370 
371 		if (CG(increment_lineno)) {
372 			CG(zend_lineno)++;
373 			CG(increment_lineno) = 0;
374 		}
375 
376 		token_line = CG(zend_lineno);
377 	}
378 
379 	zval_ptr_dtor_str(&source_zval);
380 	zend_restore_lexical_state(&original_lex_state);
381 	zend_hash_destroy(&interned_strings);
382 
383 	return 1;
384 }
385 
386 struct event_context {
387 	zval *tokens;
388 	zend_class_entry *token_class;
389 };
390 
extract_token_id_to_replace(zval * token_zv,const char * text,size_t length)391 static zval *extract_token_id_to_replace(zval *token_zv, const char *text, size_t length) {
392 	zval *id_zv, *text_zv;
393 	ZEND_ASSERT(token_zv);
394 	if (Z_TYPE_P(token_zv) == IS_ARRAY) {
395 		id_zv = zend_hash_index_find(Z_ARRVAL_P(token_zv), 0);
396 		text_zv = zend_hash_index_find(Z_ARRVAL_P(token_zv), 1);
397 	} else if (Z_TYPE_P(token_zv) == IS_OBJECT) {
398 		id_zv = OBJ_PROP_NUM(Z_OBJ_P(token_zv), 0);
399 		text_zv = OBJ_PROP_NUM(Z_OBJ_P(token_zv), 1);
400 	} else {
401 		return NULL;
402 	}
403 
404 	/* There are multiple candidate tokens to which this feedback may apply,
405 	 * check text to make sure this is the right one. */
406 	ZEND_ASSERT(Z_TYPE_P(text_zv) == IS_STRING);
407 	if (Z_STRLEN_P(text_zv) == length && !memcmp(Z_STRVAL_P(text_zv), text, length)) {
408 		return id_zv;
409 	}
410 	return NULL;
411 }
412 
on_event(zend_php_scanner_event event,int token,int line,const char * text,size_t length,void * context)413 void on_event(
414 		zend_php_scanner_event event, int token, int line,
415 		const char *text, size_t length, void *context)
416 {
417 	struct event_context *ctx = context;
418 
419 	switch (event) {
420 		case ON_TOKEN:
421 			if (token == END) break;
422 			/* Special cases */
423 			if (token == ';' && LANG_SCNG(yy_leng) > 1) { /* ?> or ?>\n or ?>\r\n */
424 				token = T_CLOSE_TAG;
425 			} else if (token == T_ECHO && LANG_SCNG(yy_leng) == sizeof("<?=") - 1) {
426 				token = T_OPEN_TAG_WITH_ECHO;
427 			}
428 			add_token(
429 				ctx->tokens, token, (unsigned char *) text, length, line, ctx->token_class, NULL);
430 			break;
431 		case ON_FEEDBACK: {
432 			HashTable *tokens_ht = Z_ARRVAL_P(ctx->tokens);
433 			zval *token_zv, *id_zv = NULL;
434 			ZEND_HASH_REVERSE_FOREACH_VAL(tokens_ht, token_zv) {
435 				id_zv = extract_token_id_to_replace(token_zv, text, length);
436 				if (id_zv) {
437 					break;
438 				}
439 			} ZEND_HASH_FOREACH_END();
440 			ZEND_ASSERT(id_zv);
441 			ZVAL_LONG(id_zv, token);
442 			break;
443 		}
444 		case ON_STOP:
445 			if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
446 				add_token(ctx->tokens, T_INLINE_HTML, LANG_SCNG(yy_cursor),
447 					LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno),
448 					ctx->token_class, NULL);
449 			}
450 			break;
451 	}
452 }
453 
tokenize_parse(zval * return_value,zend_string * source,zend_class_entry * token_class)454 static bool tokenize_parse(
455 		zval *return_value, zend_string *source, zend_class_entry *token_class)
456 {
457 	zval source_zval;
458 	struct event_context ctx;
459 	zval token_stream;
460 	zend_lex_state original_lex_state;
461 	bool original_in_compilation;
462 	bool success;
463 
464 	ZVAL_STR_COPY(&source_zval, source);
465 
466 	original_in_compilation = CG(in_compilation);
467 	CG(in_compilation) = 1;
468 	zend_save_lexical_state(&original_lex_state);
469 
470 	zend_prepare_string_for_scanning(&source_zval, ZSTR_EMPTY_ALLOC());
471 	array_init(&token_stream);
472 
473 	ctx.tokens = &token_stream;
474 	ctx.token_class = token_class;
475 
476 	CG(ast) = NULL;
477 	CG(ast_arena) = zend_arena_create(1024 * 32);
478 	LANG_SCNG(yy_state) = yycINITIAL;
479 	LANG_SCNG(on_event) = on_event;
480 	LANG_SCNG(on_event_context) = &ctx;
481 
482 	if((success = (zendparse() == SUCCESS))) {
483 		ZVAL_COPY_VALUE(return_value, &token_stream);
484 	} else {
485 		zval_ptr_dtor(&token_stream);
486 	}
487 
488 	zend_ast_destroy(CG(ast));
489 	zend_arena_destroy(CG(ast_arena));
490 
491 	/* restore compiler and scanner global states */
492 	zend_restore_lexical_state(&original_lex_state);
493 	CG(in_compilation) = original_in_compilation;
494 
495 	zval_ptr_dtor_str(&source_zval);
496 
497 	return success;
498 }
499 
tokenize_common(zval * return_value,zend_string * source,zend_long flags,zend_class_entry * token_class)500 static bool tokenize_common(
501 		zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class)
502 {
503 	if (flags & TOKEN_PARSE) {
504 		return tokenize_parse(return_value, source, token_class);
505 	} else {
506 		int success = tokenize(return_value, source, token_class);
507 		/* Normal token_get_all() should not throw. */
508 		zend_clear_exception();
509 		return success;
510 	}
511 }
512 
513 /* }}} */
514 
515 /* {{{ */
PHP_FUNCTION(token_get_all)516 PHP_FUNCTION(token_get_all)
517 {
518 	zend_string *source;
519 	zend_long flags = 0;
520 
521 	ZEND_PARSE_PARAMETERS_START(1, 2)
522 		Z_PARAM_STR(source)
523 		Z_PARAM_OPTIONAL
524 		Z_PARAM_LONG(flags)
525 	ZEND_PARSE_PARAMETERS_END();
526 
527 	if (!tokenize_common(return_value, source, flags, /* token_class */ NULL)) {
528 		RETURN_THROWS();
529 	}
530 }
531 /* }}} */
532 
533 /* {{{ */
PHP_FUNCTION(token_name)534 PHP_FUNCTION(token_name)
535 {
536 	zend_long type;
537 
538 	ZEND_PARSE_PARAMETERS_START(1, 1)
539 		Z_PARAM_LONG(type)
540 	ZEND_PARSE_PARAMETERS_END();
541 
542 	const char *token_name = get_token_type_name(type);
543 	if (!token_name) {
544 		token_name = "UNKNOWN";
545 	}
546 	RETURN_STRING(token_name);
547 }
548 /* }}} */
549