1 /*
2 +----------------------------------------------------------------------+
3 | PHP Version 5 |
4 +----------------------------------------------------------------------+
5 | Copyright (c) 1997-2014 The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Author: Andrei Zmievski <andrei@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 /* $Id$ */
20
21 #ifdef HAVE_CONFIG_H
22 #include "config.h"
23 #endif
24
25 #include "php.h"
26 #include "php_ini.h"
27 #include "ext/standard/info.h"
28 #include "php_tokenizer.h"
29
30 #include "zend.h"
31 #include "zend_language_scanner.h"
32 #include "zend_language_scanner_defs.h"
33 #include <zend_language_parser.h>
34
35 #define zendtext LANG_SCNG(yy_text)
36 #define zendleng LANG_SCNG(yy_leng)
37 #define zendcursor LANG_SCNG(yy_cursor)
38 #define zendlimit LANG_SCNG(yy_limit)
39
40 /* {{{ arginfo */
41 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_get_all, 0, 0, 1)
42 ZEND_ARG_INFO(0, source)
43 ZEND_END_ARG_INFO()
44
45 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_name, 0, 0, 1)
46 ZEND_ARG_INFO(0, token)
47 ZEND_END_ARG_INFO()
48 /* }}} */
49
50 /* {{{ tokenizer_functions[]
51 *
52 * Every user visible function must have an entry in tokenizer_functions[].
53 */
54 const zend_function_entry tokenizer_functions[] = {
55 PHP_FE(token_get_all, arginfo_token_get_all)
56 PHP_FE(token_name, arginfo_token_name)
57 PHP_FE_END
58 };
59 /* }}} */
60
61 /* {{{ tokenizer_module_entry
62 */
63 zend_module_entry tokenizer_module_entry = {
64 #if ZEND_MODULE_API_NO >= 20010901
65 STANDARD_MODULE_HEADER,
66 #endif
67 "tokenizer",
68 tokenizer_functions,
69 PHP_MINIT(tokenizer),
70 NULL,
71 NULL,
72 NULL,
73 PHP_MINFO(tokenizer),
74 #if ZEND_MODULE_API_NO >= 20010901
75 "0.1", /* Replace with version number for your extension */
76 #endif
77 STANDARD_MODULE_PROPERTIES
78 };
79 /* }}} */
80
81 #ifdef COMPILE_DL_TOKENIZER
82 ZEND_GET_MODULE(tokenizer)
83 #endif
84
85 /* {{{ PHP_MINIT_FUNCTION
86 */
PHP_MINIT_FUNCTION(tokenizer)87 PHP_MINIT_FUNCTION(tokenizer)
88 {
89 tokenizer_register_constants(INIT_FUNC_ARGS_PASSTHRU);
90 return SUCCESS;
91 }
92 /* }}} */
93
94 /* {{{ PHP_MINFO_FUNCTION
95 */
PHP_MINFO_FUNCTION(tokenizer)96 PHP_MINFO_FUNCTION(tokenizer)
97 {
98 php_info_print_table_start();
99 php_info_print_table_row(2, "Tokenizer Support", "enabled");
100 php_info_print_table_end();
101 }
102 /* }}} */
103
tokenize(zval * return_value TSRMLS_DC)104 static void tokenize(zval *return_value TSRMLS_DC)
105 {
106 zval token;
107 zval *keyword;
108 int token_type;
109 zend_bool destroy;
110 int token_line = 1;
111 int need_tokens = -1; // for __halt_compiler lexing. -1 = disabled
112
113 array_init(return_value);
114
115 ZVAL_NULL(&token);
116 while ((token_type = lex_scan(&token TSRMLS_CC))) {
117 destroy = 1;
118 switch (token_type) {
119 case T_CLOSE_TAG:
120 if (zendtext[zendleng - 1] != '>') {
121 CG(zend_lineno)++;
122 }
123 case T_OPEN_TAG:
124 case T_OPEN_TAG_WITH_ECHO:
125 case T_WHITESPACE:
126 case T_COMMENT:
127 case T_DOC_COMMENT:
128 destroy = 0;
129 break;
130 }
131
132 if (token_type >= 256) {
133 MAKE_STD_ZVAL(keyword);
134 array_init(keyword);
135 add_next_index_long(keyword, token_type);
136 if (token_type == T_END_HEREDOC) {
137 if (CG(increment_lineno)) {
138 token_line = ++CG(zend_lineno);
139 CG(increment_lineno) = 0;
140 }
141 add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1);
142 efree(Z_STRVAL(token));
143 } else {
144 add_next_index_stringl(keyword, (char *)zendtext, zendleng, 1);
145 }
146 add_next_index_long(keyword, token_line);
147 add_next_index_zval(return_value, keyword);
148 } else {
149 add_next_index_stringl(return_value, (char *)zendtext, zendleng, 1);
150 }
151 if (destroy && Z_TYPE(token) != IS_NULL) {
152 zval_dtor(&token);
153 }
154 ZVAL_NULL(&token);
155
156 // after T_HALT_COMPILER collect the next three non-dropped tokens
157 if (need_tokens != -1) {
158 if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
159 && token_type != T_COMMENT && token_type != T_DOC_COMMENT
160 && --need_tokens == 0
161 ) {
162 // fetch the rest into a T_INLINE_HTML
163 if (zendcursor != zendlimit) {
164 MAKE_STD_ZVAL(keyword);
165 array_init(keyword);
166 add_next_index_long(keyword, T_INLINE_HTML);
167 add_next_index_stringl(keyword, (char *)zendcursor, zendlimit - zendcursor, 1);
168 add_next_index_long(keyword, token_line);
169 add_next_index_zval(return_value, keyword);
170 }
171 break;
172 }
173 } else if (token_type == T_HALT_COMPILER) {
174 need_tokens = 3;
175 }
176
177 token_line = CG(zend_lineno);
178 }
179 }
180
181 /* {{{ proto array token_get_all(string source)
182 */
PHP_FUNCTION(token_get_all)183 PHP_FUNCTION(token_get_all)
184 {
185 char *source = NULL;
186 int argc = ZEND_NUM_ARGS();
187 int source_len;
188 zval source_z;
189 zend_lex_state original_lex_state;
190
191 if (zend_parse_parameters(argc TSRMLS_CC, "s", &source, &source_len) == FAILURE) {
192 return;
193 }
194
195 ZVAL_STRINGL(&source_z, source, source_len, 1);
196 zend_save_lexical_state(&original_lex_state TSRMLS_CC);
197
198 if (zend_prepare_string_for_scanning(&source_z, "" TSRMLS_CC) == FAILURE) {
199 zend_restore_lexical_state(&original_lex_state TSRMLS_CC);
200 RETURN_FALSE;
201 }
202
203 LANG_SCNG(yy_state) = yycINITIAL;
204
205 tokenize(return_value TSRMLS_CC);
206
207 zend_restore_lexical_state(&original_lex_state TSRMLS_CC);
208 zval_dtor(&source_z);
209 }
210 /* }}} */
211
212 /* {{{ proto string token_name(int type)
213 */
PHP_FUNCTION(token_name)214 PHP_FUNCTION(token_name)
215 {
216 int argc = ZEND_NUM_ARGS();
217 long type;
218
219 if (zend_parse_parameters(argc TSRMLS_CC, "l", &type) == FAILURE) {
220 return;
221 }
222 RETVAL_STRING(get_token_type_name(type), 1);
223 }
224 /* }}} */
225
226 /*
227 * Local variables:
228 * tab-width: 4
229 * c-basic-offset: 4
230 * End:
231 * vim600: noet sw=4 ts=4 fdm=marker
232 * vim<600: noet sw=4 ts=4
233 */
234
