xref: /PHP-7.2/ext/tokenizer/tokenizer.c (revision 7a7ec01a) 
1 /*
2    +----------------------------------------------------------------------+
3    | PHP Version 7                                                        |
4    +----------------------------------------------------------------------+
5    | Copyright (c) 1997-2018 The PHP Group                                |
6    +----------------------------------------------------------------------+
7    | This source file is subject to version 3.01 of the PHP license,      |
8    | that is bundled with this package in the file LICENSE, and is        |
9    | available through the world-wide-web at the following url:           |
10    | http://www.php.net/license/3_01.txt                                  |
11    | If you did not receive a copy of the PHP license and are unable to   |
12    | obtain it through the world-wide-web, please send a note to          |
13    | license@php.net so we can mail you a copy immediately.               |
14    +----------------------------------------------------------------------+
15    | Author: Andrei Zmievski <andrei@php.net>                             |
16    +----------------------------------------------------------------------+
17 */
18 
19 /* $Id$ */
20 
21 #ifdef HAVE_CONFIG_H
22 #include "config.h"
23 #endif
24 
25 #include "php.h"
26 #include "php_ini.h"
27 #include "ext/standard/info.h"
28 #include "php_tokenizer.h"
29 
30 #include "zend.h"
31 #include "zend_exceptions.h"
32 #include "zend_language_scanner.h"
33 #include "zend_language_scanner_defs.h"
34 #include <zend_language_parser.h>
35 
36 #define zendtext   LANG_SCNG(yy_text)
37 #define zendleng   LANG_SCNG(yy_leng)
38 #define zendcursor LANG_SCNG(yy_cursor)
39 #define zendlimit  LANG_SCNG(yy_limit)
40 
41 #define TOKEN_PARSE 				1
42 
tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS)43 void tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS) {
44 	REGISTER_LONG_CONSTANT("TOKEN_PARSE", TOKEN_PARSE, CONST_CS|CONST_PERSISTENT);
45 }
46 
47 /* {{{ arginfo */
48 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_get_all, 0, 0, 1)
49 	ZEND_ARG_INFO(0, source)
50 	ZEND_ARG_INFO(0, flags)
51 ZEND_END_ARG_INFO()
52 
53 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_name, 0, 0, 1)
54 	ZEND_ARG_INFO(0, token)
55 ZEND_END_ARG_INFO()
56 /* }}} */
57 
58 /* {{{ tokenizer_functions[]
59  *
60  * Every user visible function must have an entry in tokenizer_functions[].
61  */
62 const zend_function_entry tokenizer_functions[] = {
63 	PHP_FE(token_get_all,	arginfo_token_get_all)
64 	PHP_FE(token_name,		arginfo_token_name)
65 	PHP_FE_END
66 };
67 /* }}} */
68 
69 /* {{{ tokenizer_module_entry
70  */
71 zend_module_entry tokenizer_module_entry = {
72 	STANDARD_MODULE_HEADER,
73 	"tokenizer",
74 	tokenizer_functions,
75 	PHP_MINIT(tokenizer),
76 	NULL,
77 	NULL,
78 	NULL,
79 	PHP_MINFO(tokenizer),
80 	PHP_TOKENIZER_VERSION,
81 	STANDARD_MODULE_PROPERTIES
82 };
83 /* }}} */
84 
85 #ifdef COMPILE_DL_TOKENIZER
86 ZEND_GET_MODULE(tokenizer)
87 #endif
88 
89 /* {{{ PHP_MINIT_FUNCTION
90  */
PHP_MINIT_FUNCTION(tokenizer)91 PHP_MINIT_FUNCTION(tokenizer)
92 {
93 	tokenizer_register_constants(INIT_FUNC_ARGS_PASSTHRU);
94 	tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS_PASSTHRU);
95 	return SUCCESS;
96 }
97 /* }}} */
98 
99 /* {{{ PHP_MINFO_FUNCTION
100  */
PHP_MINFO_FUNCTION(tokenizer)101 PHP_MINFO_FUNCTION(tokenizer)
102 {
103 	php_info_print_table_start();
104 	php_info_print_table_row(2, "Tokenizer Support", "enabled");
105 	php_info_print_table_end();
106 }
107 /* }}} */
108 
add_token(zval * return_value,int token_type,unsigned char * text,size_t leng,int lineno)109 static void add_token(zval *return_value, int token_type,
110 		unsigned char *text, size_t leng, int lineno) {
111 	if (token_type >= 256) {
112 		zval keyword;
113 		array_init(&keyword);
114 		add_next_index_long(&keyword, token_type);
115 		add_next_index_stringl(&keyword, (char *) text, leng);
116 		add_next_index_long(&keyword, lineno);
117 		add_next_index_zval(return_value, &keyword);
118 	} else {
119 		if (leng == 1) {
120 			add_next_index_str(return_value, ZSTR_CHAR(text[0]));
121 		} else {
122 			add_next_index_stringl(return_value, (char *) text, leng);
123 		}
124 	}
125 }
126 
tokenize(zval * return_value,zend_string * source)127 static zend_bool tokenize(zval *return_value, zend_string *source)
128 {
129 	zval source_zval;
130 	zend_lex_state original_lex_state;
131 	zval token;
132 	int token_type;
133 	int token_line = 1;
134 	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
135 
136 	ZVAL_STR_COPY(&source_zval, source);
137 	zend_save_lexical_state(&original_lex_state);
138 
139 	if (zend_prepare_string_for_scanning(&source_zval, "") == FAILURE) {
140 		zend_restore_lexical_state(&original_lex_state);
141 		return 0;
142 	}
143 
144 	LANG_SCNG(yy_state) = yycINITIAL;
145 	array_init(return_value);
146 
147 	ZVAL_UNDEF(&token);
148 	while ((token_type = lex_scan(&token))) {
149 		add_token(return_value, token_type, zendtext, zendleng, token_line);
150 
151 		if (Z_TYPE(token) != IS_UNDEF) {
152 			zval_dtor(&token);
153 			ZVAL_UNDEF(&token);
154 		}
155 
156 		/* after T_HALT_COMPILER collect the next three non-dropped tokens */
157 		if (need_tokens != -1) {
158 			if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
159 				&& token_type != T_COMMENT && token_type != T_DOC_COMMENT
160 				&& --need_tokens == 0
161 			) {
162 				/* fetch the rest into a T_INLINE_HTML */
163 				if (zendcursor != zendlimit) {
164 					add_token(return_value, T_INLINE_HTML,
165 						zendcursor, zendlimit - zendcursor, token_line);
166 				}
167 				break;
168 			}
169 		} else if (token_type == T_HALT_COMPILER) {
170 			need_tokens = 3;
171 		}
172 
173 		if (CG(increment_lineno)) {
174 			CG(zend_lineno)++;
175 			CG(increment_lineno) = 0;
176 		}
177 
178 		token_line = CG(zend_lineno);
179 	}
180 
181 	zval_dtor(&source_zval);
182 	zend_restore_lexical_state(&original_lex_state);
183 
184 	return 1;
185 }
186 
on_event(zend_php_scanner_event event,int token,int line,void * context)187 void on_event(zend_php_scanner_event event, int token, int line, void *context)
188 {
189 	zval *token_stream = (zval *) context;
190 	HashTable *tokens_ht;
191 	zval *token_zv;
192 
193 	switch (event) {
194 		case ON_TOKEN:
195 			if (token == END) break;
196 			add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
197 			break;
198 		case ON_FEEDBACK:
199 			tokens_ht = Z_ARRVAL_P(token_stream);
200 			token_zv = zend_hash_index_find(tokens_ht, zend_hash_num_elements(tokens_ht) - 1);
201 			if (token_zv && Z_TYPE_P(token_zv) == IS_ARRAY) {
202 				ZVAL_LONG(zend_hash_index_find(Z_ARRVAL_P(token_zv), 0), token);
203 			}
204 			break;
205 		case ON_STOP:
206 			if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
207 				add_token(token_stream, T_INLINE_HTML, LANG_SCNG(yy_cursor),
208 					LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno));
209 			}
210 			break;
211 	}
212 }
213 
tokenize_parse(zval * return_value,zend_string * source)214 static zend_bool tokenize_parse(zval *return_value, zend_string *source)
215 {
216 	zval source_zval;
217 	zend_lex_state original_lex_state;
218 	zend_bool original_in_compilation;
219 	zend_bool success;
220 
221 	ZVAL_STR_COPY(&source_zval, source);
222 
223 	original_in_compilation = CG(in_compilation);
224 	CG(in_compilation) = 1;
225 	zend_save_lexical_state(&original_lex_state);
226 
227 	if ((success = (zend_prepare_string_for_scanning(&source_zval, "") == SUCCESS))) {
228 		zval token_stream;
229 		array_init(&token_stream);
230 
231 		CG(ast) = NULL;
232 		CG(ast_arena) = zend_arena_create(1024 * 32);
233 		LANG_SCNG(yy_state) = yycINITIAL;
234 		LANG_SCNG(on_event) = on_event;
235 		LANG_SCNG(on_event_context) = &token_stream;
236 
237 		if((success = (zendparse() == SUCCESS))) {
238 			ZVAL_COPY_VALUE(return_value, &token_stream);
239 		} else {
240 			zval_ptr_dtor(&token_stream);
241 		}
242 
243 		zend_ast_destroy(CG(ast));
244 		zend_arena_destroy(CG(ast_arena));
245 	}
246 
247 	/* restore compiler and scanner global states */
248 	zend_restore_lexical_state(&original_lex_state);
249 	CG(in_compilation) = original_in_compilation;
250 
251 	zval_dtor(&source_zval);
252 
253 	return success;
254 }
255 
256 /* }}} */
257 
258 /* {{{ proto array token_get_all(string source [, int flags])
259  */
PHP_FUNCTION(token_get_all)260 PHP_FUNCTION(token_get_all)
261 {
262 	zend_string *source;
263 	zend_long flags = 0;
264 	zend_bool success;
265 
266 	ZEND_PARSE_PARAMETERS_START(1, 2)
267 		Z_PARAM_STR(source)
268 		Z_PARAM_OPTIONAL
269 		Z_PARAM_LONG(flags)
270 	ZEND_PARSE_PARAMETERS_END();
271 
272 	if (flags & TOKEN_PARSE) {
273 		success = tokenize_parse(return_value, source);
274 	} else {
275 		success = tokenize(return_value, source);
276 		/* Normal token_get_all() should not throw. */
277 		zend_clear_exception();
278 	}
279 
280 	if (!success) RETURN_FALSE;
281 }
282 /* }}} */
283 
284 /* {{{ proto string token_name(int type)
285  */
PHP_FUNCTION(token_name)286 PHP_FUNCTION(token_name)
287 {
288 	zend_long type;
289 
290 	ZEND_PARSE_PARAMETERS_START(1, 1)
291 		Z_PARAM_LONG(type)
292 	ZEND_PARSE_PARAMETERS_END();
293 
294 	RETVAL_STRING(get_token_type_name(type));
295 }
296 /* }}} */
297 
298 /*
299  * Local variables:
300  * tab-width: 4
301  * c-basic-offset: 4
302  * End:
303  * vim600: noet sw=4 ts=4 fdm=marker
304  * vim<600: noet sw=4 ts=4
305  */
306