1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 | Xinchen Hui <laruence@php.net> |
17 +----------------------------------------------------------------------+
18 */
19
20 #include "Zend/zend_execute.h"
21 #include "Zend/zend_exceptions.h"
22 #include "Zend/zend_vm.h"
23 #include "Zend/zend_closures.h"
24 #include "Zend/zend_constants.h"
25 #include "Zend/zend_API.h"
26
27 #include <ZendAccelerator.h>
28 #include "Optimizer/zend_func_info.h"
29 #include "Optimizer/zend_call_graph.h"
30 #include "zend_jit.h"
31
32 #include "zend_jit_internal.h"
33
34 #ifdef HAVE_GCC_GLOBAL_REGS
35 # pragma GCC diagnostic ignored "-Wvolatile-register-var"
36 # if defined(__x86_64__)
37 register zend_execute_data* volatile execute_data __asm__("%r14");
38 register const zend_op* volatile opline __asm__("%r15");
39 # elif defined(i386)
40 register zend_execute_data* volatile execute_data __asm__("%esi");
41 register const zend_op* volatile opline __asm__("%edi");
42 # elif defined(__aarch64__)
43 register zend_execute_data* volatile execute_data __asm__("x27");
44 register const zend_op* volatile opline __asm__("x28");
45 # endif
46 # pragma GCC diagnostic warning "-Wvolatile-register-var"
47 #endif
48
zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)49 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)
50 {
51 zend_execute_data *old_execute_data;
52
53 if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
54 zend_clean_and_cache_symbol_table(EX(symbol_table));
55 }
56
57 zend_vm_stack_free_extra_args_ex(call_info, execute_data);
58 if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) {
59 OBJ_RELEASE(Z_OBJ(execute_data->This));
60 } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
61 OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
62 }
63 if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
64 zend_free_extra_named_params(EX(extra_named_params));
65 }
66
67 old_execute_data = execute_data;
68 execute_data = EX(prev_execute_data);
69 zend_vm_stack_free_call_frame_ex(call_info, old_execute_data);
70
71 if (UNEXPECTED(EG(exception) != NULL)) {
72 const zend_op *old_opline = EX(opline);
73 zend_throw_exception_internal(NULL);
74 if (old_opline->result_type != IS_UNDEF) {
75 zval_ptr_dtor(EX_VAR(old_opline->result.var));
76 }
77 #ifndef HAVE_GCC_GLOBAL_REGS
78 return 2; // ZEND_VM_LEAVE
79 #endif
80 } else {
81 EX(opline)++;
82 #ifdef HAVE_GCC_GLOBAL_REGS
83 opline = EX(opline);
84 #else
85 return 2; // ZEND_VM_LEAVE
86 #endif
87 }
88 }
89
zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)90 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)
91 {
92 if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) {
93 if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
94 zend_clean_and_cache_symbol_table(EX(symbol_table));
95 }
96 zend_vm_stack_free_extra_args_ex(call_info, execute_data);
97 }
98 if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
99 zend_free_extra_named_params(EX(extra_named_params));
100 }
101 if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
102 OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
103 }
104 execute_data = EG(current_execute_data);
105 #ifdef HAVE_GCC_GLOBAL_REGS
106 opline = zend_jit_halt_op;
107 #else
108 return -1; // ZEND_VM_RETURN
109 #endif
110 }
111
zend_jit_leave_func_helper(EXECUTE_DATA_D)112 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(EXECUTE_DATA_D)
113 {
114 uint32_t call_info = EX_CALL_INFO();
115
116 if (call_info & ZEND_CALL_TOP) {
117 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info);
118 } else {
119 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info);
120 }
121 }
122
zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)123 void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)
124 {
125 zend_op_array *op_array = &EX(func)->op_array;
126
127 if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) {
128 uint32_t first_extra_arg = op_array->num_args;
129 uint32_t num_args = EX_NUM_ARGS();
130 zval *end, *src, *dst;
131 uint32_t type_flags = 0;
132
133 if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) {
134 /* Skip useless ZEND_RECV and ZEND_RECV_INIT opcodes */
135 #ifdef HAVE_GCC_GLOBAL_REGS
136 opline += first_extra_arg;
137 #else
138 EX(opline) += first_extra_arg;
139 #endif
140 }
141
142 /* move extra args into separate array after all CV and TMP vars */
143 end = EX_VAR_NUM(first_extra_arg - 1);
144 src = end + (num_args - first_extra_arg);
145 dst = src + (op_array->last_var + op_array->T - first_extra_arg);
146 if (EXPECTED(src != dst)) {
147 do {
148 type_flags |= Z_TYPE_INFO_P(src);
149 ZVAL_COPY_VALUE(dst, src);
150 ZVAL_UNDEF(src);
151 src--;
152 dst--;
153 } while (src != end);
154 if (type_flags & (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT)) {
155 ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
156 }
157 } else {
158 do {
159 if (Z_REFCOUNTED_P(src)) {
160 ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
161 break;
162 }
163 src--;
164 } while (src != end);
165 }
166 }
167 }
168
zend_jit_deprecated_helper(OPLINE_D)169 bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D)
170 {
171 zend_execute_data *call = (zend_execute_data *) opline;
172 zend_function *fbc = call->func;
173
174 zend_deprecated_function(fbc);
175
176 if (EG(exception)) {
177 #ifndef HAVE_GCC_GLOBAL_REGS
178 zend_execute_data *execute_data = EG(current_execute_data);
179 #endif
180 const zend_op *opline = EG(opline_before_exception);
181 if (opline && RETURN_VALUE_USED(opline)) {
182 ZVAL_UNDEF(EX_VAR(opline->result.var));
183 }
184
185 zend_vm_stack_free_args(call);
186
187 if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) {
188 OBJ_RELEASE(Z_OBJ(call->This));
189 }
190
191 zend_vm_stack_free_call_frame(call);
192 return 0;
193 }
194 return 1;
195 }
196
zend_jit_undefined_long_key(EXECUTE_DATA_D)197 void ZEND_FASTCALL zend_jit_undefined_long_key(EXECUTE_DATA_D)
198 {
199 const zend_op *opline = EX(opline);
200 zval *result = EX_VAR(opline->result.var);
201 zval *dim;
202
203 if (opline->op2_type == IS_CONST) {
204 dim = RT_CONSTANT(opline, opline->op2);
205 } else {
206 dim = EX_VAR(opline->op2.var);
207 }
208 ZEND_ASSERT(Z_TYPE_P(dim) == IS_LONG);
209 zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, Z_LVAL_P(dim));
210 ZVAL_NULL(result);
211 }
212
zend_jit_undefined_string_key(EXECUTE_DATA_D)213 void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D)
214 {
215 const zend_op *opline = EX(opline);
216 zval *result = EX_VAR(opline->result.var);
217 zval *dim;
218 zend_ulong lval;
219
220 if (opline->op2_type == IS_CONST) {
221 dim = RT_CONSTANT(opline, opline->op2);
222 } else {
223 dim = EX_VAR(opline->op2.var);
224 }
225 ZEND_ASSERT(Z_TYPE_P(dim) == IS_STRING);
226 if (ZEND_HANDLE_NUMERIC(Z_STR_P(dim), lval)) {
227 zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, lval);
228 } else {
229 zend_error(E_WARNING, "Undefined array key \"%s\"", Z_STRVAL_P(dim));
230 }
231 ZVAL_NULL(result);
232 }
233
zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)234 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)
235 {
236 zend_op_array *op_array = (zend_op_array*)EX(func);
237 zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
238 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t) jit_extension->orig_handler;
239 ++*(uintptr_t*)(EX(run_time_cache) + zend_jit_profile_counter_rid);
240 ++zend_jit_profile_counter;
241 ZEND_OPCODE_TAIL_CALL(handler);
242 }
243
zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)244 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
245 {
246 zend_jit_op_array_hot_extension *jit_extension =
247 (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
248 #ifndef HAVE_GCC_GLOBAL_REGS
249 const zend_op *opline = EX(opline);
250 #endif
251
252 *(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func));
253
254 if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
255 *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
256 zend_jit_hot_func(execute_data, opline);
257 ZEND_OPCODE_RETURN();
258 } else {
259 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
260 ZEND_OPCODE_TAIL_CALL(handler);
261 }
262 }
263
zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)264 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
265 {
266 zend_jit_op_array_hot_extension *jit_extension =
267 (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
268 #ifndef HAVE_GCC_GLOBAL_REGS
269 const zend_op *opline = EX(opline);
270 #endif
271
272 *(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop));
273
274 if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
275 *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
276 zend_jit_hot_func(execute_data, opline);
277 ZEND_OPCODE_RETURN();
278 } else {
279 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
280 ZEND_OPCODE_TAIL_CALL(handler);
281 }
282 }
283
_zend_quick_get_constant(const zval * key,uint32_t flags,int check_defined_only)284 static zend_always_inline zend_constant* _zend_quick_get_constant(
285 const zval *key, uint32_t flags, int check_defined_only)
286 {
287 #ifndef HAVE_GCC_GLOBAL_REGS
288 zend_execute_data *execute_data = EG(current_execute_data);
289 #endif
290 const zend_op *opline = EX(opline);
291 zval *zv;
292 zend_constant *c = NULL;
293
294 /* null/true/false are resolved during compilation, so don't check for them here. */
295 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
296 if (zv) {
297 c = (zend_constant*)Z_PTR_P(zv);
298 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
299 key++;
300 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
301 if (zv) {
302 c = (zend_constant*)Z_PTR_P(zv);
303 }
304 }
305
306 if (!c) {
307 if (!check_defined_only) {
308 zend_throw_error(NULL, "Undefined constant \"%s\"", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)));
309 ZVAL_UNDEF(EX_VAR(opline->result.var));
310 }
311 CACHE_PTR(opline->extended_value, ENCODE_SPECIAL_CACHE_NUM(zend_hash_num_elements(EG(zend_constants))));
312 return NULL;
313 }
314
315 if (!check_defined_only) {
316 if (ZEND_CONSTANT_FLAGS(c) & CONST_DEPRECATED) {
317 zend_error(E_DEPRECATED, "Constant %s is deprecated", ZSTR_VAL(c->name));
318 if (EG(exception)) {
319 return NULL;
320 }
321 return c;
322 }
323 }
324
325 CACHE_PTR(opline->extended_value, c);
326 return c;
327 }
328
zend_jit_get_constant(const zval * key,uint32_t flags)329 zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags)
330 {
331 return _zend_quick_get_constant(key, flags, 0);
332 }
333
zend_jit_check_constant(const zval * key)334 zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key)
335 {
336 return _zend_quick_get_constant(key, 0, 1);
337 }
338
zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)339 static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)
340 {
341 zend_jit_op_array_trace_extension *jit_extension =
342 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
343 size_t offset = jit_extension->offset;
344 #ifndef HAVE_GCC_GLOBAL_REGS
345 const zend_op *opline = EX(opline);
346 #endif
347
348 *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
349
350 if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
351 *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT;
352 if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
353 #ifdef HAVE_GCC_GLOBAL_REGS
354 opline = NULL;
355 return;
356 #else
357 return -1;
358 #endif
359 }
360 #ifdef HAVE_GCC_GLOBAL_REGS
361 execute_data = EG(current_execute_data);
362 opline = execute_data ? EX(opline) : NULL;
363 return;
364 #else
365 return 1;
366 #endif
367 } else {
368 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
369 ZEND_OPCODE_TAIL_CALL(handler);
370 }
371 }
372
zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)373 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
374 {
375 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
376 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
377 }
378
zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)379 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
380 {
381 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
382 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
383 }
384
zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)385 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
386 {
387 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
388 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
389 }
390
391 #define TRACE_RECORD(_op, _info, _ptr) \
392 trace_buffer[idx].info = _op | (_info); \
393 trace_buffer[idx].ptr = _ptr; \
394 idx++; \
395 if (idx >= JIT_G(max_trace_length) - 2) { \
396 stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
397 break; \
398 }
399
400 #define TRACE_RECORD_VM(_op, _ptr, _op1_type, _op2_type, _op3_type) \
401 trace_buffer[idx].op = _op; \
402 trace_buffer[idx].op1_type = _op1_type; \
403 trace_buffer[idx].op2_type = _op2_type; \
404 trace_buffer[idx].op3_type = _op3_type; \
405 trace_buffer[idx].ptr = _ptr; \
406 idx++; \
407 if (idx >= JIT_G(max_trace_length) - 2) { \
408 stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
409 break; \
410 }
411
412 #define TRACE_START(_op, _start, _ptr1, _ptr2) \
413 trace_buffer[0].op = _op; \
414 trace_buffer[0].start = _start; \
415 trace_buffer[0].level = 0; \
416 trace_buffer[0].ptr = _ptr1; \
417 trace_buffer[1].last = 0; \
418 trace_buffer[1].ptr = _ptr2; \
419 idx = ZEND_JIT_TRACE_START_REC_SIZE;
420
421 #define TRACE_END(_op, _stop, _ptr) \
422 trace_buffer[1].last = idx; \
423 trace_buffer[idx].op = _op; \
424 trace_buffer[idx].start = trace_buffer[idx].start; \
425 trace_buffer[idx].stop = trace_buffer[0].stop = _stop; \
426 trace_buffer[idx].level = trace_buffer[0].level = ret_level ? ret_level + 1 : 0; \
427 trace_buffer[idx].ptr = _ptr;
428
zend_jit_trace_recursive_call_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level,int level)429 static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level)
430 {
431 int i;
432 int count = 0;
433
434 for (i = ret_level; i < level; i++) {
435 count += (unrolled_calls[i] == op_array);
436 }
437 return count;
438 }
439
zend_jit_trace_recursive_ret_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level)440 static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level)
441 {
442 int i;
443 int count = 0;
444
445 for (i = 0; i < ret_level; i++) {
446 count += (unrolled_calls[i] == op_array);
447 }
448 return count;
449 }
450
zend_jit_trace_has_recursive_ret(zend_execute_data * ex,const zend_op_array * orig_op_array,const zend_op * orig_opline,int ret_level)451 static int zend_jit_trace_has_recursive_ret(zend_execute_data *ex, const zend_op_array *orig_op_array, const zend_op *orig_opline, int ret_level)
452 {
453 while (ex != NULL && ex->func != NULL && ret_level < ZEND_JIT_TRACE_MAX_RET_DEPTH) {
454 if (&ex->func->op_array == orig_op_array && ex->opline + 1 == orig_opline) {
455 return 1;
456 }
457 ex = ex->prev_execute_data;
458 ret_level++;
459 }
460 return 0;
461 }
462
zend_jit_trace_bad_stop_event(const zend_op * opline,int count)463 static uint8_t zend_jit_trace_bad_stop_event(const zend_op *opline, int count)
464 {
465 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
466 uint8_t *cache_count = JIT_G(bad_root_cache_count);
467 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
468 uint32_t i;
469
470 if (count < 0) {
471 count = 0;
472 }
473 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
474 if (cache_opline[i] == opline) {
475 if (cache_count[i] >= count) {
476 return cache_stop[i];
477 }
478 break;
479 }
480 }
481 return 0;
482 }
483
484 #define ZEND_CALL_MEGAMORPHIC ZEND_CALL_JIT_RESERVED
485
zend_jit_trace_record_fake_init_call_ex(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic,uint32_t init_level)486 static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t init_level)
487 {
488 zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
489
490 do {
491 zend_function *func;
492 zend_jit_op_array_trace_extension *jit_extension;
493
494 if (call->prev_execute_data) {
495 idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, init_level + 1);
496 if (idx < 0) {
497 return idx;
498 }
499 }
500
501 func = call->func;
502 if (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE|ZEND_ACC_NEVER_CACHE)) {
503 /* TODO: Can we continue recording ??? */
504 return -1;
505 }
506 if (func->type == ZEND_INTERNAL_FUNCTION
507 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
508 return -1;
509 }
510 if (func->type == ZEND_USER_FUNCTION
511 && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
512 jit_extension =
513 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
514 if (UNEXPECTED(!jit_extension
515 || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)
516 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE))) {
517 return -1;
518 }
519 func = (zend_function*)jit_extension->op_array;
520 }
521 if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM
522 /* TODO: use more accurate check ??? */
523 && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
524 || func->common.scope)) {
525 func = NULL;
526 ZEND_ADD_CALL_FLAG(call, ZEND_CALL_MEGAMORPHIC);
527 }
528 TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INFO(init_level), func);
529 } while (0);
530 return idx;
531 }
532
zend_jit_trace_record_fake_init_call(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic)533 static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic)
534 {
535 return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, 0);
536 }
537
zend_jit_trace_subtrace(zend_jit_trace_rec * trace_buffer,int start,int end,uint8_t event,const zend_op_array * op_array,const zend_op * opline)538 static int zend_jit_trace_subtrace(zend_jit_trace_rec *trace_buffer, int start, int end, uint8_t event, const zend_op_array *op_array, const zend_op *opline)
539 {
540 int idx;
541
542 TRACE_START(ZEND_JIT_TRACE_START, event, op_array, opline);
543 memmove(trace_buffer + idx, trace_buffer + start, (end - start) * sizeof(zend_jit_trace_rec));
544 return idx + (end - start);
545 }
546
547 /*
548 * Trace Linking Rules
549 * ===================
550 *
551 * flags
552 * +----------+----------+----------++----------+----------+----------+
553 * | || JIT |
554 * +----------+----------+----------++----------+----------+----------+
555 * start | LOOP | ENTER | RETURN || LOOP | ENTER | RETURN |
556 * +========+==========+==========+==========++==========+==========+==========+
557 * | LOOP | loop | | loop-ret || COMPILED | LINK | LINK |
558 * +--------+----------+----------+----------++----------+----------+----------+
559 * | ENTER |INNER_LOOP| rec-call | return || LINK | LINK | LINK |
560 * +--------+----------+----------+----------++----------+----------+----------+
561 * | RETURN |INNER_LOOP| | rec-ret || LINK | | LINK |
562 * +--------+----------+----------+----------++----------+----------+----------+
563 * | SIDE | unroll | | return || LINK | LINK | LINK |
564 * +--------+----------+----------+----------++----------+----------+----------+
565 *
566 * loop: LOOP if "cycle" and level == 0, otherwise INNER_LOOP
567 * INNER_LOOP: abort recording and start new one (wait for loop)
568 * COMPILED: abort recording (wait while side exit creates outer loop)
569 * unroll: continue recording while unroll limit reached
570 * rec-call: RECURSIVE_CALL if "cycle" and level > N, otherwise continue
571 * loop-ret: LOOP_EXIT if level == 0, otherwise continue (wait for loop)
572 * return: RETURN if level == 0
573 * rec_ret: RECURSIVE_RET if "cycle" and ret_level > N, otherwise continue
574 *
575 */
576
zend_jit_trace_execute(zend_execute_data * ex,const zend_op * op,zend_jit_trace_rec * trace_buffer,uint8_t start,uint32_t is_megamorphic)577 zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphic)
578
579 {
580 #ifdef HAVE_GCC_GLOBAL_REGS
581 zend_execute_data *save_execute_data = execute_data;
582 const zend_op *save_opline = opline;
583 #endif
584 const zend_op *orig_opline, *end_opline;
585 zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
586 zend_jit_trace_stop halt = 0;
587 int level = 0;
588 int ret_level = 0;
589 zend_vm_opcode_handler_t handler;
590 const zend_op_array *op_array;
591 zend_jit_op_array_trace_extension *jit_extension;
592 size_t offset;
593 int idx, count;
594 uint8_t trace_flags, op1_type, op2_type, op3_type;
595 zend_class_entry *ce1, *ce2;
596 const zend_op *link_to_enter_opline = NULL;
597 int backtrack_link_to_enter = -1;
598 int backtrack_recursion = -1;
599 int backtrack_ret_recursion = -1;
600 int backtrack_ret_recursion_level = 0;
601 int loop_unroll_limit = 0;
602 int last_loop = -1;
603 int last_loop_level = -1;
604 const zend_op *last_loop_opline = NULL;
605 const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
606 #ifdef HAVE_GCC_GLOBAL_REGS
607 zend_execute_data *prev_execute_data = ex;
608
609 execute_data = ex;
610 opline = EX(opline) = op;
611 #else
612 int rc;
613 zend_execute_data *execute_data = ex;
614 const zend_op *opline = EX(opline);
615 #endif
616 zend_execute_data *prev_call = EX(call);
617
618 orig_opline = opline;
619
620 op_array = &EX(func)->op_array;
621 jit_extension =
622 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
623 offset = jit_extension->offset;
624 if (!op_array->function_name
625 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
626 op_array = jit_extension->op_array;
627 }
628
629 TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline);
630
631 if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
632 /* Abort trace because of exception */
633 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_EXCEPTION, opline);
634 #ifdef HAVE_GCC_GLOBAL_REGS
635 execute_data = save_execute_data;
636 opline = save_opline;
637 #endif
638 return ZEND_JIT_TRACE_STOP_EXCEPTION;
639 }
640
641 trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
642 if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
643 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_NOT_SUPPORTED, opline);
644 #ifdef HAVE_GCC_GLOBAL_REGS
645 execute_data = save_execute_data;
646 opline = save_opline;
647 #endif
648 return ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
649 }
650
651 if (prev_call) {
652 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic);
653 if (ret < 0) {
654 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_BAD_FUNC, opline);
655 #ifdef HAVE_GCC_GLOBAL_REGS
656 execute_data = save_execute_data;
657 opline = save_opline;
658 #endif
659 return ZEND_JIT_TRACE_STOP_BAD_FUNC;
660 }
661 idx = ret;
662 }
663
664 while (1) {
665 ce1 = ce2 = NULL;
666 op1_type = op2_type = op3_type = IS_UNKNOWN;
667 if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV))
668 && opline->opcode != ZEND_ROPE_ADD
669 && opline->opcode != ZEND_ROPE_END
670 && opline->opcode != ZEND_NEW
671 && opline->opcode != ZEND_FETCH_CLASS_CONSTANT
672 && opline->opcode != ZEND_INIT_STATIC_METHOD_CALL) {
673 zval *zv = EX_VAR(opline->op1.var);
674 op1_type = Z_TYPE_P(zv);
675 uint8_t flags = 0;
676
677 if (op1_type == IS_INDIRECT) {
678 zv = Z_INDIRECT_P(zv);
679 op1_type = Z_TYPE_P(zv);
680 flags |= IS_TRACE_INDIRECT;
681 }
682 if (op1_type == IS_REFERENCE) {
683 zv = Z_REFVAL_P(zv);
684 op1_type = Z_TYPE_P(zv);
685 flags |= IS_TRACE_REFERENCE;
686 }
687 if (Z_TYPE_P(zv) == IS_OBJECT) {
688 ce1 = Z_OBJCE_P(zv);
689 } else if (Z_TYPE_P(zv) == IS_ARRAY) {
690 if (HT_IS_PACKED(Z_ARRVAL_P(zv))) {
691 flags |= IS_TRACE_PACKED;
692 }
693 }
694 op1_type |= flags;
695 } else if (opline->op1_type == IS_UNUSED && (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
696 uint32_t op1_flags = ZEND_VM_OP1_FLAGS(zend_get_opcode_flags(opline->opcode));
697 if ((op1_flags & ZEND_VM_OP_MASK) == ZEND_VM_OP_THIS) {
698 op1_type = IS_OBJECT;
699 ce1 = Z_OBJCE(EX(This));
700 }
701 }
702 if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)
703 && opline->opcode != ZEND_INSTANCEOF
704 && opline->opcode != ZEND_UNSET_STATIC_PROP
705 && opline->opcode != ZEND_ISSET_ISEMPTY_STATIC_PROP
706 && opline->opcode != ZEND_ASSIGN_STATIC_PROP
707 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_REF
708 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_OP
709 && opline->opcode != ZEND_PRE_INC_STATIC_PROP
710 && opline->opcode != ZEND_POST_INC_STATIC_PROP
711 && opline->opcode != ZEND_PRE_DEC_STATIC_PROP
712 && opline->opcode != ZEND_POST_DEC_STATIC_PROP
713 && opline->opcode != ZEND_FETCH_STATIC_PROP_R
714 && opline->opcode != ZEND_FETCH_STATIC_PROP_W
715 && opline->opcode != ZEND_FETCH_STATIC_PROP_RW
716 && opline->opcode != ZEND_FETCH_STATIC_PROP_IS
717 && opline->opcode != ZEND_FETCH_STATIC_PROP_FUNC_ARG
718 && opline->opcode != ZEND_FETCH_STATIC_PROP_UNSET
719 && (opline->op2_type == IS_CV
720 || (opline->opcode != ZEND_FE_FETCH_R
721 && opline->opcode != ZEND_FE_FETCH_RW))) {
722 zval *zv = EX_VAR(opline->op2.var);
723 uint8_t flags = 0;
724
725 op2_type = Z_TYPE_P(zv);
726 if (op2_type == IS_INDIRECT) {
727 zv = Z_INDIRECT_P(zv);
728 op2_type = Z_TYPE_P(zv);
729 flags |= IS_TRACE_INDIRECT;
730 }
731 if (op2_type == IS_REFERENCE) {
732 zv = Z_REFVAL_P(zv);
733 op2_type = Z_TYPE_P(zv);
734 flags |= IS_TRACE_REFERENCE;
735 }
736 if (Z_TYPE_P(zv) == IS_OBJECT) {
737 ce2 = Z_OBJCE_P(zv);
738 }
739 op2_type |= flags;
740 }
741 if (opline->opcode == ZEND_ASSIGN_DIM ||
742 opline->opcode == ZEND_ASSIGN_OBJ ||
743 opline->opcode == ZEND_ASSIGN_STATIC_PROP ||
744 opline->opcode == ZEND_ASSIGN_DIM_OP ||
745 opline->opcode == ZEND_ASSIGN_OBJ_OP ||
746 opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP ||
747 opline->opcode == ZEND_ASSIGN_OBJ_REF ||
748 opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF) {
749 if ((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
750 zval *zv = EX_VAR((opline+1)->op1.var);
751 uint8_t flags = 0;
752
753 op3_type = Z_TYPE_P(zv);
754 if (op3_type == IS_INDIRECT) {
755 zv = Z_INDIRECT_P(zv);
756 op3_type = Z_TYPE_P(zv);
757 flags |= IS_TRACE_INDIRECT;
758 }
759 if (op3_type == IS_REFERENCE) {
760 zv = Z_REFVAL_P(zv);
761 op3_type = Z_TYPE_P(zv);
762 flags |= IS_TRACE_REFERENCE;
763 }
764 op3_type |= flags;
765 }
766 }
767
768 TRACE_RECORD_VM(ZEND_JIT_TRACE_VM, opline, op1_type, op2_type, op3_type);
769
770 if (ce1) {
771 TRACE_RECORD(ZEND_JIT_TRACE_OP1_TYPE, 0, ce1);
772 }
773
774 if (ce2) {
775 TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2);
776 }
777
778 switch (opline->opcode) {
779 case ZEND_FETCH_DIM_R:
780 case ZEND_FETCH_DIM_W:
781 case ZEND_FETCH_DIM_RW:
782 case ZEND_FETCH_DIM_IS:
783 case ZEND_FETCH_DIM_FUNC_ARG:
784 case ZEND_FETCH_DIM_UNSET:
785 case ZEND_FETCH_LIST_R:
786 case ZEND_FETCH_LIST_W:
787 case ZEND_ASSIGN_DIM:
788 case ZEND_ASSIGN_DIM_OP:
789 case ZEND_UNSET_DIM:
790 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
791 if (opline->op1_type == IS_CONST) {
792 zval *arr = RT_CONSTANT(opline, opline->op1);
793 op1_type = Z_TYPE_P(arr);
794 }
795 if ((op1_type & IS_TRACE_TYPE_MASK) == IS_ARRAY
796 && opline->op2_type != IS_UNDEF) {
797 zval *arr, *dim, *val;
798 uint8_t val_type = IS_UNDEF;
799
800 if (opline->op2_type == IS_CONST) {
801 dim = RT_CONSTANT(opline, opline->op2);
802 } else {
803 dim = EX_VAR(opline->op2.var);
804 }
805
806 if (Z_TYPE_P(dim) == IS_LONG || Z_TYPE_P(dim) == IS_STRING) {
807 if (opline->op1_type == IS_CONST) {
808 arr = RT_CONSTANT(opline, opline->op1);
809 } else {
810 arr = EX_VAR(opline->op1.var);
811 }
812 if (Z_TYPE_P(arr) == IS_INDIRECT) {
813 arr = Z_INDIRECT_P(arr);
814 }
815 if (Z_TYPE_P(arr) == IS_REFERENCE) {
816 arr = Z_REFVAL_P(arr);
817 }
818 ZEND_ASSERT(Z_TYPE_P(arr) == IS_ARRAY);
819 if (Z_TYPE_P(dim) == IS_LONG) {
820 val = zend_hash_index_find(Z_ARRVAL_P(arr), Z_LVAL_P(dim));
821 } else /*if Z_TYPE_P(dim) == IS_STRING)*/ {
822 val = zend_symtable_find(Z_ARRVAL_P(arr), Z_STR_P(dim));
823 }
824 if (val) {
825 val_type = Z_TYPE_P(val);
826 }
827 TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, val_type, 0, 0);
828 }
829 }
830 break;
831 case ZEND_FETCH_OBJ_R:
832 case ZEND_FETCH_OBJ_W:
833 case ZEND_FETCH_OBJ_RW:
834 case ZEND_FETCH_OBJ_IS:
835 case ZEND_FETCH_OBJ_FUNC_ARG:
836 case ZEND_FETCH_OBJ_UNSET:
837 case ZEND_ASSIGN_OBJ:
838 case ZEND_ASSIGN_OBJ_OP:
839 case ZEND_ASSIGN_OBJ_REF:
840 case ZEND_UNSET_OBJ:
841 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
842 case ZEND_PRE_INC_OBJ:
843 case ZEND_PRE_DEC_OBJ:
844 case ZEND_POST_INC_OBJ:
845 case ZEND_POST_DEC_OBJ:
846 if (opline->op1_type != IS_CONST
847 && opline->op2_type == IS_CONST
848 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
849 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
850 zval *obj, *val;
851 zend_string *prop_name = Z_STR_P(RT_CONSTANT(opline, opline->op2));
852 zend_property_info *prop_info;
853
854 if (opline->op1_type == IS_UNUSED) {
855 obj = &EX(This);
856 } else {
857 obj = EX_VAR(opline->op1.var);
858 }
859 if (Z_TYPE_P(obj) != IS_OBJECT
860 || Z_OBJ_P(obj)->handlers != &std_object_handlers) {
861 break;
862 }
863 prop_info = zend_get_property_info(Z_OBJCE_P(obj), prop_name, 1);
864 if (prop_info
865 && prop_info != ZEND_WRONG_PROPERTY_INFO
866 && !(prop_info->flags & ZEND_ACC_STATIC)) {
867 val = OBJ_PROP(Z_OBJ_P(obj), prop_info->offset);
868 TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, Z_TYPE_P(val), 0, 0);
869 }
870 }
871 break;
872 default:
873 break;
874 }
875
876 if (opline->opcode == ZEND_DO_FCALL
877 || opline->opcode == ZEND_DO_ICALL
878 || opline->opcode == ZEND_DO_UCALL
879 || opline->opcode == ZEND_DO_FCALL_BY_NAME) {
880 if (ZEND_CALL_INFO(EX(call)) & ZEND_CALL_MEGAMORPHIC) {
881 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
882 break;
883 }
884 if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
885 if (EX(call)->func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) {
886 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
887 break;
888 }
889 TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func);
890 }
891 } else if (opline->opcode == ZEND_INCLUDE_OR_EVAL
892 || opline->opcode == ZEND_CALLABLE_CONVERT) {
893 /* TODO: Support tracing JIT for ZEND_CALLABLE_CONVERT. */
894 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
895 break;
896 }
897
898 handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
899 #ifdef HAVE_GCC_GLOBAL_REGS
900 handler();
901 if (UNEXPECTED(opline == zend_jit_halt_op)) {
902 stop = ZEND_JIT_TRACE_STOP_RETURN;
903 opline = NULL;
904 halt = ZEND_JIT_TRACE_HALT;
905 break;
906 }
907 if (UNEXPECTED(execute_data != prev_execute_data)) {
908 #else
909 rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);
910 if (rc != 0) {
911 if (rc < 0) {
912 stop = ZEND_JIT_TRACE_STOP_RETURN;
913 opline = NULL;
914 halt = ZEND_JIT_TRACE_HALT;
915 break;
916 } else if (execute_data == EG(current_execute_data)) {
917 /* return after interrupt handler */
918 rc = 0;
919 }
920 execute_data = EG(current_execute_data);
921 opline = EX(opline);
922 #endif
923
924 op_array = &EX(func)->op_array;
925 jit_extension =
926 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
927 if (UNEXPECTED(!jit_extension)
928 || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) {
929 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
930 break;
931 }
932 offset = jit_extension->offset;
933 if (!op_array->function_name
934 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
935 op_array = jit_extension->op_array;
936 }
937
938 #ifdef HAVE_GCC_GLOBAL_REGS
939 if (execute_data->prev_execute_data == prev_execute_data) {
940 #else
941 if (rc == 0) {
942 /* pass */
943 } else if (rc == 1) {
944 #endif
945 /* Enter into function */
946 prev_call = NULL;
947 if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) {
948 stop = ZEND_JIT_TRACE_STOP_TOO_DEEP;
949 break;
950 }
951
952 if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
953 /* TODO: Can we continue recording ??? */
954 stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
955 break;
956 }
957
958 TRACE_RECORD(ZEND_JIT_TRACE_ENTER,
959 EX(return_value) != NULL ? ZEND_JIT_TRACE_RETURN_VALUE_USED : 0,
960 op_array);
961
962 count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level);
963
964 if (opline == orig_opline) {
965 if (count + 1 >= JIT_G(max_recursive_calls)) {
966 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
967 break;
968 }
969 backtrack_recursion = idx;
970 } else if (count >= JIT_G(max_recursive_calls)) {
971 stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
972 break;
973 }
974
975 unrolled_calls[ret_level + level] = &EX(func)->op_array;
976 level++;
977 } else {
978 /* Return from function */
979 prev_call = EX(call);
980 if (level == 0) {
981 if (start == ZEND_JIT_TRACE_START_RETURN
982 && JIT_G(max_recursive_returns) > 0
983 && execute_data->prev_execute_data
984 && execute_data->prev_execute_data->func
985 && execute_data->prev_execute_data->func->type == ZEND_USER_FUNCTION
986 && zend_jit_trace_has_recursive_ret(execute_data, trace_buffer[0].op_array, orig_opline, ret_level)) {
987 if (ret_level > ZEND_JIT_TRACE_MAX_RET_DEPTH) {
988 stop = ZEND_JIT_TRACE_STOP_TOO_DEEP_RET;
989 break;
990 }
991 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
992 count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level);
993 if (opline == orig_opline) {
994 if (count + 1 >= JIT_G(max_recursive_returns)) {
995 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
996 break;
997 }
998 backtrack_ret_recursion = idx;
999 backtrack_ret_recursion_level = ret_level;
1000 } else if (count >= JIT_G(max_recursive_returns)) {
1001 stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
1002 break;
1003 }
1004
1005 unrolled_calls[ret_level] = &EX(func)->op_array;
1006 ret_level++;
1007 last_loop_opline = NULL;
1008
1009 if (prev_call) {
1010 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0);
1011 if (ret < 0) {
1012 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1013 break;
1014 }
1015 idx = ret;
1016 }
1017 } else if (start & ZEND_JIT_TRACE_START_LOOP
1018 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1019 ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
1020 /* Fail to try close the loop.
1021 If this doesn't work terminate it. */
1022 stop = ZEND_JIT_TRACE_STOP_LOOP_EXIT;
1023 break;
1024 } else if (start & ZEND_JIT_TRACE_START_ENTER
1025 && EX(prev_execute_data)
1026 && EX(func) == EX(prev_execute_data)->func
1027 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1028 ZEND_JIT_TRACE_STOP_RECURSION_EXIT) {
1029 stop = ZEND_JIT_TRACE_STOP_RECURSION_EXIT;
1030 break;
1031 } else {
1032 stop = ZEND_JIT_TRACE_STOP_RETURN;
1033 break;
1034 }
1035 } else {
1036 level--;
1037 if (level < last_loop_level) {
1038 last_loop_opline = NULL;
1039 }
1040 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
1041 }
1042 }
1043 #ifdef HAVE_GCC_GLOBAL_REGS
1044 prev_execute_data = execute_data;
1045 #endif
1046 }
1047 if (EX(call) != prev_call) {
1048 if (EX(call)
1049 && EX(call)->prev_execute_data == prev_call) {
1050 zend_function *func;
1051 zend_jit_op_array_trace_extension *jit_extension;
1052
1053 if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
1054 /* TODO: Can we continue recording ??? */
1055 stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
1056 break;
1057 } else if (EX(call)->func->common.fn_flags & ZEND_ACC_NEVER_CACHE) {
1058 /* TODO: Can we continue recording ??? */
1059 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1060 break;
1061 }
1062 func = EX(call)->func;
1063 if (func->type == ZEND_INTERNAL_FUNCTION
1064 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
1065 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1066 break;
1067 }
1068 if (func->type == ZEND_USER_FUNCTION
1069 && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
1070 jit_extension =
1071 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
1072 if (UNEXPECTED(!jit_extension)
1073 || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)
1074 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) {
1075 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
1076 break;
1077 }
1078 func = (zend_function*)jit_extension->op_array;
1079 }
1080
1081 #ifndef HAVE_GCC_GLOBAL_REGS
1082 opline = EX(opline);
1083 #endif
1084
1085 if (JIT_G(max_polymorphic_calls) == 0
1086 && zend_jit_may_be_polymorphic_call(opline - 1)) {
1087 func = NULL;
1088 } else if ((is_megamorphic == ZEND_JIT_EXIT_METHOD_CALL
1089 || is_megamorphic == ZEND_JIT_EXIT_CLOSURE_CALL)
1090 && trace_buffer[1].opline == opline - 1) {
1091 func = NULL;
1092 }
1093 if (!func) {
1094 ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC);
1095 }
1096 TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func);
1097 }
1098 prev_call = EX(call);
1099 }
1100
1101 #ifndef HAVE_GCC_GLOBAL_REGS
1102 opline = EX(opline);
1103 #endif
1104
1105 if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
1106 /* Abort trace because of exception */
1107 stop = ZEND_JIT_TRACE_STOP_EXCEPTION;
1108 break;
1109 }
1110
1111 trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
1112 if (trace_flags) {
1113 if (trace_flags & ZEND_JIT_TRACE_JITED) {
1114 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
1115 if ((start & ZEND_JIT_TRACE_START_LOOP) != 0
1116 && level + ret_level == 0
1117 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1118 ZEND_JIT_TRACE_STOP_COMPILED_LOOP) {
1119 /* Fail to try close outer loop through side exit.
1120 If this doesn't work just link. */
1121 stop = ZEND_JIT_TRACE_STOP_COMPILED_LOOP;
1122 break;
1123 } else {
1124 stop = ZEND_JIT_TRACE_STOP_LINK;
1125 break;
1126 }
1127 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
1128 if (start != ZEND_JIT_TRACE_START_RETURN) {
1129 // TODO: We may try to inline function ???
1130 stop = ZEND_JIT_TRACE_STOP_LINK;
1131 break;
1132 }
1133 if (backtrack_link_to_enter < 0) {
1134 backtrack_link_to_enter = idx;
1135 link_to_enter_opline = opline;
1136 }
1137 } else {
1138 stop = ZEND_JIT_TRACE_STOP_LINK;
1139 break;
1140 }
1141 } else if (trace_flags & ZEND_JIT_TRACE_BLACKLISTED) {
1142 stop = ZEND_JIT_TRACE_STOP_BLACK_LIST;
1143 break;
1144 } else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
1145 uint8_t bad_stop;
1146
1147 if (start != ZEND_JIT_TRACE_START_SIDE) {
1148 if (opline == orig_opline && level + ret_level == 0) {
1149 stop = ZEND_JIT_TRACE_STOP_LOOP;
1150 break;
1151 }
1152 }
1153
1154 if (start != ZEND_JIT_TRACE_START_SIDE
1155 || level + ret_level != 0) {
1156 /* First try creating a trace for inner loop.
1157 If this doesn't work try loop unroling. */
1158 bad_stop = zend_jit_trace_bad_stop_event(opline,
1159 JIT_G(blacklist_root_trace) / 2);
1160 if (bad_stop != ZEND_JIT_TRACE_STOP_INNER_LOOP
1161 && bad_stop != ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
1162 if (start == ZEND_JIT_TRACE_START_SIDE
1163 || zend_jit_trace_bad_stop_event(orig_opline,
1164 JIT_G(blacklist_root_trace) / 2) != ZEND_JIT_TRACE_STOP_INNER_LOOP) {
1165 stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
1166 break;
1167 }
1168 }
1169 }
1170
1171 if (opline == last_loop_opline
1172 && level == last_loop_level) {
1173 idx = zend_jit_trace_subtrace(trace_buffer,
1174 last_loop, idx, ZEND_JIT_TRACE_START_LOOP, op_array, opline);
1175 start = ZEND_JIT_TRACE_START_LOOP;
1176 stop = ZEND_JIT_TRACE_STOP_LOOP;
1177 ret_level = 0;
1178 break;
1179 } else if (loop_unroll_limit < JIT_G(max_loop_unrolls)) {
1180 last_loop = idx;
1181 last_loop_opline = opline;
1182 last_loop_level = level;
1183 loop_unroll_limit++;
1184 } else {
1185 stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL;
1186 break;
1187 }
1188 } else if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
1189 TRACE_RECORD(ZEND_JIT_TRACE_VM, 0, opline);
1190 stop = ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
1191 break;
1192 }
1193 }
1194 }
1195
1196 end_opline = opline;
1197 if (!ZEND_JIT_TRACE_STOP_OK(stop)) {
1198 if (backtrack_recursion > 0) {
1199 idx = backtrack_recursion;
1200 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
1201 end_opline = orig_opline;
1202 } else if (backtrack_ret_recursion > 0) {
1203 idx = backtrack_ret_recursion;
1204 ret_level = backtrack_ret_recursion_level;
1205 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
1206 end_opline = orig_opline;
1207 } else if (backtrack_link_to_enter > 0) {
1208 if (stop == ZEND_JIT_TRACE_STOP_DEEP_RECURSION
1209 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) / 2) ==
1210 ZEND_JIT_TRACE_STOP_DEEP_RECURSION) {
1211 idx = backtrack_link_to_enter;
1212 stop = ZEND_JIT_TRACE_STOP_LINK;
1213 end_opline = link_to_enter_opline;
1214 }
1215 }
1216 }
1217
1218 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
1219 /* Shrink fake INIT_CALLs */
1220 while (trace_buffer[idx-1].op == ZEND_JIT_TRACE_INIT_CALL
1221 && (trace_buffer[idx-1].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1222 idx--;
1223 }
1224 }
1225
1226 TRACE_END(ZEND_JIT_TRACE_END, stop, end_opline);
1227
1228 #ifdef HAVE_GCC_GLOBAL_REGS
1229 if (!halt) {
1230 EX(opline) = opline;
1231 }
1232 #endif
1233
1234 #ifdef HAVE_GCC_GLOBAL_REGS
1235 execute_data = save_execute_data;
1236 opline = save_opline;
1237 #endif
1238
1239 return stop | halt;
1240 }
1241
