xref: /PHP-7.3/ext/tokenizer/tokenizer.c (revision 8d3f8ca1) 
1 /*
2    +----------------------------------------------------------------------+
3    | PHP Version 7                                                        |
4    +----------------------------------------------------------------------+
5    | Copyright (c) 1997-2018 The PHP Group                                |
6    +----------------------------------------------------------------------+
7    | This source file is subject to version 3.01 of the PHP license,      |
8    | that is bundled with this package in the file LICENSE, and is        |
9    | available through the world-wide-web at the following url:           |
10    | http://www.php.net/license/3_01.txt                                  |
11    | If you did not receive a copy of the PHP license and are unable to   |
12    | obtain it through the world-wide-web, please send a note to          |
13    | license@php.net so we can mail you a copy immediately.               |
14    +----------------------------------------------------------------------+
15    | Author: Andrei Zmievski <andrei@php.net>                             |
16    +----------------------------------------------------------------------+
17 */
18 
19 #ifdef HAVE_CONFIG_H
20 #include "config.h"
21 #endif
22 
23 #include "php.h"
24 #include "php_ini.h"
25 #include "ext/standard/info.h"
26 #include "php_tokenizer.h"
27 
28 #include "zend.h"
29 #include "zend_exceptions.h"
30 #include "zend_language_scanner.h"
31 #include "zend_language_scanner_defs.h"
32 #include <zend_language_parser.h>
33 
34 #define zendtext   LANG_SCNG(yy_text)
35 #define zendleng   LANG_SCNG(yy_leng)
36 #define zendcursor LANG_SCNG(yy_cursor)
37 #define zendlimit  LANG_SCNG(yy_limit)
38 
39 #define TOKEN_PARSE 				1
40 
tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS)41 void tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS) {
42 	REGISTER_LONG_CONSTANT("TOKEN_PARSE", TOKEN_PARSE, CONST_CS|CONST_PERSISTENT);
43 }
44 
45 /* {{{ arginfo */
46 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_get_all, 0, 0, 1)
47 	ZEND_ARG_INFO(0, source)
48 	ZEND_ARG_INFO(0, flags)
49 ZEND_END_ARG_INFO()
50 
51 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_name, 0, 0, 1)
52 	ZEND_ARG_INFO(0, token)
53 ZEND_END_ARG_INFO()
54 /* }}} */
55 
56 /* {{{ tokenizer_functions[]
57  *
58  * Every user visible function must have an entry in tokenizer_functions[].
59  */
60 static const zend_function_entry tokenizer_functions[] = {
61 	PHP_FE(token_get_all,	arginfo_token_get_all)
62 	PHP_FE(token_name,		arginfo_token_name)
63 	PHP_FE_END
64 };
65 /* }}} */
66 
67 /* {{{ tokenizer_module_entry
68  */
69 zend_module_entry tokenizer_module_entry = {
70 	STANDARD_MODULE_HEADER,
71 	"tokenizer",
72 	tokenizer_functions,
73 	PHP_MINIT(tokenizer),
74 	NULL,
75 	NULL,
76 	NULL,
77 	PHP_MINFO(tokenizer),
78 	PHP_TOKENIZER_VERSION,
79 	STANDARD_MODULE_PROPERTIES
80 };
81 /* }}} */
82 
83 #ifdef COMPILE_DL_TOKENIZER
84 ZEND_GET_MODULE(tokenizer)
85 #endif
86 
87 /* {{{ PHP_MINIT_FUNCTION
88  */
PHP_MINIT_FUNCTION(tokenizer)89 PHP_MINIT_FUNCTION(tokenizer)
90 {
91 	tokenizer_register_constants(INIT_FUNC_ARGS_PASSTHRU);
92 	tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS_PASSTHRU);
93 	return SUCCESS;
94 }
95 /* }}} */
96 
97 /* {{{ PHP_MINFO_FUNCTION
98  */
PHP_MINFO_FUNCTION(tokenizer)99 PHP_MINFO_FUNCTION(tokenizer)
100 {
101 	php_info_print_table_start();
102 	php_info_print_table_row(2, "Tokenizer Support", "enabled");
103 	php_info_print_table_end();
104 }
105 /* }}} */
106 
add_token(zval * return_value,int token_type,unsigned char * text,size_t leng,int lineno)107 static void add_token(zval *return_value, int token_type,
108 		unsigned char *text, size_t leng, int lineno) {
109 	if (token_type >= 256) {
110 		zval keyword;
111 		array_init(&keyword);
112 		add_next_index_long(&keyword, token_type);
113 		add_next_index_stringl(&keyword, (char *) text, leng);
114 		add_next_index_long(&keyword, lineno);
115 		add_next_index_zval(return_value, &keyword);
116 	} else {
117 		if (leng == 1) {
118 			add_next_index_str(return_value, ZSTR_CHAR(text[0]));
119 		} else {
120 			add_next_index_stringl(return_value, (char *) text, leng);
121 		}
122 	}
123 }
124 
tokenize(zval * return_value,zend_string * source)125 static zend_bool tokenize(zval *return_value, zend_string *source)
126 {
127 	zval source_zval;
128 	zend_lex_state original_lex_state;
129 	zval token;
130 	int token_type;
131 	int token_line = 1;
132 	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
133 
134 	ZVAL_STR_COPY(&source_zval, source);
135 	zend_save_lexical_state(&original_lex_state);
136 
137 	if (zend_prepare_string_for_scanning(&source_zval, "") == FAILURE) {
138 		zend_restore_lexical_state(&original_lex_state);
139 		return 0;
140 	}
141 
142 	LANG_SCNG(yy_state) = yycINITIAL;
143 	array_init(return_value);
144 
145 	while ((token_type = lex_scan(&token, NULL))) {
146 		add_token(return_value, token_type, zendtext, zendleng, token_line);
147 
148 		if (Z_TYPE(token) != IS_UNDEF) {
149 			zval_ptr_dtor_nogc(&token);
150 			ZVAL_UNDEF(&token);
151 		}
152 
153 		/* after T_HALT_COMPILER collect the next three non-dropped tokens */
154 		if (need_tokens != -1) {
155 			if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
156 				&& token_type != T_COMMENT && token_type != T_DOC_COMMENT
157 				&& --need_tokens == 0
158 			) {
159 				/* fetch the rest into a T_INLINE_HTML */
160 				if (zendcursor != zendlimit) {
161 					add_token(return_value, T_INLINE_HTML,
162 						zendcursor, zendlimit - zendcursor, token_line);
163 				}
164 				break;
165 			}
166 		} else if (token_type == T_HALT_COMPILER) {
167 			need_tokens = 3;
168 		}
169 
170 		if (CG(increment_lineno)) {
171 			CG(zend_lineno)++;
172 			CG(increment_lineno) = 0;
173 		}
174 
175 		token_line = CG(zend_lineno);
176 	}
177 
178 	zval_ptr_dtor_str(&source_zval);
179 	zend_restore_lexical_state(&original_lex_state);
180 
181 	return 1;
182 }
183 
on_event(zend_php_scanner_event event,int token,int line,void * context)184 void on_event(zend_php_scanner_event event, int token, int line, void *context)
185 {
186 	zval *token_stream = (zval *) context;
187 	HashTable *tokens_ht;
188 	zval *token_zv;
189 
190 	switch (event) {
191 		case ON_TOKEN:
192 			{
193 				if (token == END) break;
194 				/* Special cases */
195 				if (token == ';' && LANG_SCNG(yy_leng) > 1) { /* ?> or ?>\n or ?>\r\n */
196 					token = T_CLOSE_TAG;
197 				} else if (token == T_ECHO && LANG_SCNG(yy_leng) == sizeof("<?=") - 1) {
198 					token = T_OPEN_TAG_WITH_ECHO;
199 				}
200 				add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
201 			}
202 			break;
203 		case ON_FEEDBACK:
204 			tokens_ht = Z_ARRVAL_P(token_stream);
205 			token_zv = zend_hash_index_find(tokens_ht, zend_hash_num_elements(tokens_ht) - 1);
206 			if (token_zv && Z_TYPE_P(token_zv) == IS_ARRAY) {
207 				ZVAL_LONG(zend_hash_index_find(Z_ARRVAL_P(token_zv), 0), token);
208 			}
209 			break;
210 		case ON_STOP:
211 			if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
212 				add_token(token_stream, T_INLINE_HTML, LANG_SCNG(yy_cursor),
213 					LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno));
214 			}
215 			break;
216 	}
217 }
218 
tokenize_parse(zval * return_value,zend_string * source)219 static zend_bool tokenize_parse(zval *return_value, zend_string *source)
220 {
221 	zval source_zval;
222 	zend_lex_state original_lex_state;
223 	zend_bool original_in_compilation;
224 	zend_bool success;
225 
226 	ZVAL_STR_COPY(&source_zval, source);
227 
228 	original_in_compilation = CG(in_compilation);
229 	CG(in_compilation) = 1;
230 	zend_save_lexical_state(&original_lex_state);
231 
232 	if ((success = (zend_prepare_string_for_scanning(&source_zval, "") == SUCCESS))) {
233 		zval token_stream;
234 		array_init(&token_stream);
235 
236 		CG(ast) = NULL;
237 		CG(ast_arena) = zend_arena_create(1024 * 32);
238 		LANG_SCNG(yy_state) = yycINITIAL;
239 		LANG_SCNG(on_event) = on_event;
240 		LANG_SCNG(on_event_context) = &token_stream;
241 
242 		if((success = (zendparse() == SUCCESS))) {
243 			ZVAL_COPY_VALUE(return_value, &token_stream);
244 		} else {
245 			zval_ptr_dtor(&token_stream);
246 		}
247 
248 		zend_ast_destroy(CG(ast));
249 		zend_arena_destroy(CG(ast_arena));
250 	}
251 
252 	/* restore compiler and scanner global states */
253 	zend_restore_lexical_state(&original_lex_state);
254 	CG(in_compilation) = original_in_compilation;
255 
256 	zval_ptr_dtor_str(&source_zval);
257 
258 	return success;
259 }
260 
261 /* }}} */
262 
263 /* {{{ proto array token_get_all(string source [, int flags])
264  */
PHP_FUNCTION(token_get_all)265 PHP_FUNCTION(token_get_all)
266 {
267 	zend_string *source;
268 	zend_long flags = 0;
269 	zend_bool success;
270 
271 	ZEND_PARSE_PARAMETERS_START(1, 2)
272 		Z_PARAM_STR(source)
273 		Z_PARAM_OPTIONAL
274 		Z_PARAM_LONG(flags)
275 	ZEND_PARSE_PARAMETERS_END();
276 
277 	if (flags & TOKEN_PARSE) {
278 		success = tokenize_parse(return_value, source);
279 	} else {
280 		success = tokenize(return_value, source);
281 		/* Normal token_get_all() should not throw. */
282 		zend_clear_exception();
283 	}
284 
285 	if (!success) RETURN_FALSE;
286 }
287 /* }}} */
288 
289 /* {{{ proto string token_name(int type)
290  */
PHP_FUNCTION(token_name)291 PHP_FUNCTION(token_name)
292 {
293 	zend_long type;
294 
295 	ZEND_PARSE_PARAMETERS_START(1, 1)
296 		Z_PARAM_LONG(type)
297 	ZEND_PARSE_PARAMETERS_END();
298 
299 	RETVAL_STRING(get_token_type_name(type));
300 }
301 /* }}} */
302 
303 /*
304  * Local variables:
305  * tab-width: 4
306  * c-basic-offset: 4
307  * End:
308  * vim600: noet sw=4 ts=4 fdm=marker
309  * vim<600: noet sw=4 ts=4
310  */
311