1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 | Xinchen Hui <laruence@php.net> |
17 +----------------------------------------------------------------------+
18 */
19
20 #include "Zend/zend_execute.h"
21 #include "Zend/zend_exceptions.h"
22 #include "Zend/zend_vm.h"
23 #include "Zend/zend_closures.h"
24 #include "Zend/zend_constants.h"
25 #include "Zend/zend_API.h"
26
27 #include <ZendAccelerator.h>
28 #include "Optimizer/zend_func_info.h"
29 #include "Optimizer/zend_call_graph.h"
30 #include "zend_jit.h"
31 #include "zend_jit_x86.h"
32 #include "zend_jit_internal.h"
33
34 #ifdef HAVE_GCC_GLOBAL_REGS
35 # pragma GCC diagnostic ignored "-Wvolatile-register-var"
36 # if defined(__x86_64__)
37 register zend_execute_data* volatile execute_data __asm__("%r14");
38 register const zend_op* volatile opline __asm__("%r15");
39 # else
40 register zend_execute_data* volatile execute_data __asm__("%esi");
41 register const zend_op* volatile opline __asm__("%edi");
42 # endif
43 # pragma GCC diagnostic warning "-Wvolatile-register-var"
44 #endif
45
zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)46 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)
47 {
48 zend_execute_data *old_execute_data;
49
50 if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
51 zend_clean_and_cache_symbol_table(EX(symbol_table));
52 }
53
54 zend_vm_stack_free_extra_args_ex(call_info, execute_data);
55 if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) {
56 OBJ_RELEASE(Z_OBJ(execute_data->This));
57 } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
58 OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
59 }
60 if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
61 zend_free_extra_named_params(EX(extra_named_params));
62 }
63
64 old_execute_data = execute_data;
65 execute_data = EX(prev_execute_data);
66 zend_vm_stack_free_call_frame_ex(call_info, old_execute_data);
67
68 if (UNEXPECTED(EG(exception) != NULL)) {
69 const zend_op *old_opline = EX(opline);
70 zend_throw_exception_internal(NULL);
71 if (old_opline->result_type != IS_UNDEF) {
72 zval_ptr_dtor(EX_VAR(old_opline->result.var));
73 }
74 #ifndef HAVE_GCC_GLOBAL_REGS
75 return 2; // ZEND_VM_LEAVE
76 #endif
77 } else {
78 EX(opline)++;
79 #ifdef HAVE_GCC_GLOBAL_REGS
80 opline = EX(opline);
81 #else
82 return 2; // ZEND_VM_LEAVE
83 #endif
84 }
85 }
86
zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)87 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)
88 {
89 if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) {
90 if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
91 zend_clean_and_cache_symbol_table(EX(symbol_table));
92 }
93 zend_vm_stack_free_extra_args_ex(call_info, execute_data);
94 }
95 if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
96 zend_free_extra_named_params(EX(extra_named_params));
97 }
98 if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
99 OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
100 }
101 execute_data = EG(current_execute_data);
102 #ifdef HAVE_GCC_GLOBAL_REGS
103 opline = zend_jit_halt_op;
104 #else
105 return -1; // ZEND_VM_RETURN
106 #endif
107 }
108
zend_jit_leave_func_helper(uint32_t call_info EXECUTE_DATA_DC)109 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(uint32_t call_info EXECUTE_DATA_DC)
110 {
111 if (call_info & ZEND_CALL_TOP) {
112 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info);
113 } else {
114 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info);
115 }
116 }
117
zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)118 void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)
119 {
120 zend_op_array *op_array = &EX(func)->op_array;
121
122 if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) {
123 uint32_t first_extra_arg = op_array->num_args;
124 uint32_t num_args = EX_NUM_ARGS();
125 zval *end, *src, *dst;
126 uint32_t type_flags = 0;
127
128 if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) {
129 /* Skip useless ZEND_RECV and ZEND_RECV_INIT opcodes */
130 #ifdef HAVE_GCC_GLOBAL_REGS
131 opline += first_extra_arg;
132 #else
133 EX(opline) += first_extra_arg;
134 #endif
135 }
136
137 /* move extra args into separate array after all CV and TMP vars */
138 end = EX_VAR_NUM(first_extra_arg - 1);
139 src = end + (num_args - first_extra_arg);
140 dst = src + (op_array->last_var + op_array->T - first_extra_arg);
141 if (EXPECTED(src != dst)) {
142 do {
143 type_flags |= Z_TYPE_INFO_P(src);
144 ZVAL_COPY_VALUE(dst, src);
145 ZVAL_UNDEF(src);
146 src--;
147 dst--;
148 } while (src != end);
149 if (type_flags & (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT)) {
150 ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
151 }
152 } else {
153 do {
154 if (Z_REFCOUNTED_P(src)) {
155 ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
156 break;
157 }
158 src--;
159 } while (src != end);
160 }
161 }
162 }
163
zend_jit_deprecated_helper(OPLINE_D)164 zend_bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D)
165 {
166 zend_execute_data *call = (zend_execute_data *) opline;
167 zend_function *fbc = call->func;
168
169 zend_deprecated_function(fbc);
170
171 if (EG(exception)) {
172 #ifndef HAVE_GCC_GLOBAL_REGS
173 zend_execute_data *execute_data = EG(current_execute_data);
174 #endif
175 const zend_op *opline = EG(opline_before_exception);
176 if (RETURN_VALUE_USED(opline)) {
177 ZVAL_UNDEF(EX_VAR(opline->result.var));
178 }
179
180 zend_vm_stack_free_args(call);
181
182 if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) {
183 OBJ_RELEASE(Z_OBJ(call->This));
184 }
185
186 zend_vm_stack_free_call_frame(call);
187 return 0;
188 }
189 return 1;
190 }
191
zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)192 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)
193 {
194 zend_op_array *op_array = (zend_op_array*)EX(func);
195 zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
196 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t) jit_extension->orig_handler;
197 ++*(uintptr_t*)(EX(run_time_cache) + zend_jit_profile_counter_rid);
198 ++zend_jit_profile_counter;
199 ZEND_OPCODE_TAIL_CALL(handler);
200 }
201
zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)202 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
203 {
204 zend_jit_op_array_hot_extension *jit_extension =
205 (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
206 #ifndef HAVE_GCC_GLOBAL_REGS
207 const zend_op *opline = EX(opline);
208 #endif
209
210 *(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func));
211
212 if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
213 *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
214 zend_jit_hot_func(execute_data, opline);
215 ZEND_OPCODE_RETURN();
216 } else {
217 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
218 ZEND_OPCODE_TAIL_CALL(handler);
219 }
220 }
221
zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)222 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
223 {
224 zend_jit_op_array_hot_extension *jit_extension =
225 (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
226 #ifndef HAVE_GCC_GLOBAL_REGS
227 const zend_op *opline = EX(opline);
228 #endif
229
230 *(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop));
231
232 if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
233 *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
234 zend_jit_hot_func(execute_data, opline);
235 ZEND_OPCODE_RETURN();
236 } else {
237 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
238 ZEND_OPCODE_TAIL_CALL(handler);
239 }
240 }
241
_zend_quick_get_constant(const zval * key,uint32_t flags,int check_defined_only)242 static zend_always_inline zend_constant* _zend_quick_get_constant(
243 const zval *key, uint32_t flags, int check_defined_only)
244 {
245 #ifndef HAVE_GCC_GLOBAL_REGS
246 zend_execute_data *execute_data = EG(current_execute_data);
247 #endif
248 const zend_op *opline = EX(opline);
249 zval *zv;
250 zend_constant *c = NULL;
251
252 /* null/true/false are resolved during compilation, so don't check for them here. */
253 zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1);
254 if (zv) {
255 c = (zend_constant*)Z_PTR_P(zv);
256 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
257 key++;
258 zv = zend_hash_find_ex(EG(zend_constants), Z_STR_P(key), 1);
259 if (zv) {
260 c = (zend_constant*)Z_PTR_P(zv);
261 }
262 }
263
264 if (!c) {
265 if (!check_defined_only) {
266 zend_throw_error(NULL, "Undefined constant \"%s\"", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)));
267 ZVAL_UNDEF(EX_VAR(opline->result.var));
268 }
269 CACHE_PTR(opline->extended_value, ENCODE_SPECIAL_CACHE_NUM(zend_hash_num_elements(EG(zend_constants))));
270 return NULL;
271 }
272
273 if (!check_defined_only) {
274 if (ZEND_CONSTANT_FLAGS(c) & CONST_DEPRECATED) {
275 zend_error(E_DEPRECATED, "Constant %s is deprecated", ZSTR_VAL(c->name));
276 if (EG(exception)) {
277 return NULL;
278 }
279 return c;
280 }
281 }
282
283 CACHE_PTR(opline->extended_value, c);
284 return c;
285 }
286
zend_jit_get_constant(const zval * key,uint32_t flags)287 zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags)
288 {
289 return _zend_quick_get_constant(key, flags, 0);
290 }
291
zend_jit_check_constant(const zval * key)292 zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key)
293 {
294 return _zend_quick_get_constant(key, 0, 1);
295 }
296
zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)297 static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)
298 {
299 zend_jit_op_array_trace_extension *jit_extension =
300 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
301 size_t offset = jit_extension->offset;
302 #ifndef HAVE_GCC_GLOBAL_REGS
303 const zend_op *opline = EX(opline);
304 #endif
305
306 *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
307
308 if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
309 *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT;
310 if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
311 #ifdef HAVE_GCC_GLOBAL_REGS
312 opline = NULL;
313 return;
314 #else
315 return -1;
316 #endif
317 }
318 #ifdef HAVE_GCC_GLOBAL_REGS
319 execute_data = EG(current_execute_data);
320 opline = execute_data ? EX(opline) : NULL;
321 return;
322 #else
323 return 1;
324 #endif
325 } else {
326 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
327 ZEND_OPCODE_TAIL_CALL(handler);
328 }
329 }
330
zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)331 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
332 {
333 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
334 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
335 }
336
zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)337 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
338 {
339 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
340 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
341 }
342
zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)343 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
344 {
345 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
346 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
347 }
348
349 #define TRACE_RECORD(_op, _info, _ptr) \
350 trace_buffer[idx].info = _op | (_info); \
351 trace_buffer[idx].ptr = _ptr; \
352 idx++; \
353 if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 2) { \
354 stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
355 break; \
356 }
357
358 #define TRACE_RECORD_VM(_op, _ptr, _op1_type, _op2_type, _op3_type) \
359 trace_buffer[idx].op = _op; \
360 trace_buffer[idx].op1_type = _op1_type; \
361 trace_buffer[idx].op2_type = _op2_type; \
362 trace_buffer[idx].op3_type = _op3_type; \
363 trace_buffer[idx].ptr = _ptr; \
364 idx++; \
365 if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 2) { \
366 stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
367 break; \
368 }
369
370 #define TRACE_START(_op, _start, _ptr1, _ptr2) \
371 trace_buffer[0].op = _op; \
372 trace_buffer[0].start = _start; \
373 trace_buffer[0].level = 0; \
374 trace_buffer[0].ptr = _ptr1; \
375 trace_buffer[1].last = 0; \
376 trace_buffer[1].ptr = _ptr2; \
377 idx = ZEND_JIT_TRACE_START_REC_SIZE;
378
379 #define TRACE_END(_op, _stop, _ptr) \
380 trace_buffer[1].last = idx; \
381 trace_buffer[idx].op = _op; \
382 trace_buffer[idx].start = trace_buffer[idx].start; \
383 trace_buffer[idx].stop = trace_buffer[0].stop = _stop; \
384 trace_buffer[idx].level = trace_buffer[0].level = ret_level ? ret_level + 1 : 0; \
385 trace_buffer[idx].ptr = _ptr;
386
zend_jit_trace_recursive_call_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level,int level)387 static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level)
388 {
389 int i;
390 int count = 0;
391
392 for (i = ret_level; i < level; i++) {
393 count += (unrolled_calls[i] == op_array);
394 }
395 return count;
396 }
397
zend_jit_trace_recursive_ret_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level)398 static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level)
399 {
400 int i;
401 int count = 0;
402
403 for (i = 0; i < ret_level; i++) {
404 count += (unrolled_calls[i] == op_array);
405 }
406 return count;
407 }
408
zend_jit_trace_has_recursive_ret(zend_execute_data * ex,const zend_op_array * orig_op_array,const zend_op * orig_opline,int ret_level)409 static int zend_jit_trace_has_recursive_ret(zend_execute_data *ex, const zend_op_array *orig_op_array, const zend_op *orig_opline, int ret_level)
410 {
411 while (ex != NULL && ex->func != NULL && ret_level < ZEND_JIT_TRACE_MAX_RET_DEPTH) {
412 if (&ex->func->op_array == orig_op_array && ex->opline + 1 == orig_opline) {
413 return 1;
414 }
415 ex = ex->prev_execute_data;
416 ret_level++;
417 }
418 return 0;
419 }
420
zend_jit_trace_bad_stop_event(const zend_op * opline,int count)421 static uint8_t zend_jit_trace_bad_stop_event(const zend_op *opline, int count)
422 {
423 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
424 uint8_t *cache_count = JIT_G(bad_root_cache_count);
425 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
426 uint32_t i;
427
428 if (count < 0) {
429 count = 0;
430 }
431 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
432 if (cache_opline[i] == opline) {
433 if (cache_count[i] >= count) {
434 return cache_stop[i];
435 }
436 break;
437 }
438 }
439 return 0;
440 }
441
442 /* Workaround for PHP-8.0 */
443 #ifndef ZEND_CALL_JIT_RESERVED
444 # define ZEND_CALL_JIT_RESERVED (1<<29)
445 #endif
446
447 #define ZEND_CALL_MEGAMORPHIC ZEND_CALL_JIT_RESERVED
448
zend_jit_trace_record_fake_init_call_ex(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic,uint32_t init_level)449 static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t init_level)
450 {
451 zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
452
453 do {
454 zend_function *func;
455 zend_jit_op_array_trace_extension *jit_extension;
456
457 if (call->prev_execute_data) {
458 idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, init_level + 1);
459 if (idx < 0) {
460 return idx;
461 }
462 }
463
464 func = call->func;
465 if (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE|ZEND_ACC_NEVER_CACHE)) {
466 /* TODO: Can we continue recording ??? */
467 return -1;
468 }
469 if (func->type == ZEND_INTERNAL_FUNCTION
470 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
471 return -1;
472 }
473 if (func->type == ZEND_USER_FUNCTION
474 && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
475 jit_extension =
476 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
477 if (UNEXPECTED(!jit_extension
478 || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)
479 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE))) {
480 return -1;
481 }
482 func = (zend_function*)jit_extension->op_array;
483 }
484 if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM
485 /* TODO: use more accurate check ??? */
486 && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
487 || func->common.scope)) {
488 func = NULL;
489 ZEND_ADD_CALL_FLAG(call, ZEND_CALL_MEGAMORPHIC);
490 }
491 TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INFO(init_level), func);
492 } while (0);
493 return idx;
494 }
495
zend_jit_trace_record_fake_init_call(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic)496 static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic)
497 {
498 return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, 0);
499 }
500
zend_jit_trace_subtrace(zend_jit_trace_rec * trace_buffer,int start,int end,uint8_t event,const zend_op_array * op_array,const zend_op * opline)501 static int zend_jit_trace_subtrace(zend_jit_trace_rec *trace_buffer, int start, int end, uint8_t event, const zend_op_array *op_array, const zend_op *opline)
502 {
503 int idx;
504
505 TRACE_START(ZEND_JIT_TRACE_START, event, op_array, opline);
506 memmove(trace_buffer + idx, trace_buffer + start, (end - start) * sizeof(zend_jit_trace_rec));
507 return idx + (end - start);
508 }
509
510 /*
511 * Trace Linking Rules
512 * ===================
513 *
514 * flags
515 * +----------+----------+----------++----------+----------+----------+
516 * | || JIT |
517 * +----------+----------+----------++----------+----------+----------+
518 * start | LOOP | ENTER | RETURN || LOOP | ENTER | RETURN |
519 * +========+==========+==========+==========++==========+==========+==========+
520 * | LOOP | loop | | loop-ret || COMPILED | LINK | LINK |
521 * +--------+----------+----------+----------++----------+----------+----------+
522 * | ENTER |INNER_LOOP| rec-call | return || LINK | LINK | LINK |
523 * +--------+----------+----------+----------++----------+----------+----------+
524 * | RETURN |INNER_LOOP| | rec-ret || LINK | | LINK |
525 * +--------+----------+----------+----------++----------+----------+----------+
526 * | SIDE | unroll | | return || LINK | LINK | LINK |
527 * +--------+----------+----------+----------++----------+----------+----------+
528 *
529 * loop: LOOP if "cycle" and level == 0, otherwise INNER_LOOP
530 * INNER_LOOP: abort recording and start new one (wait for loop)
531 * COMPILED: abort recording (wait while side exit creates outer loop)
532 * unroll: continue recording while unroll limit reached
533 * rec-call: RECURSIVE_CALL if "cycle" and level > N, otherwise continue
534 * loop-ret: LOOP_EXIT if level == 0, otherwise continue (wait for loop)
535 * return: RETURN if level == 0
536 * rec_ret: RECURSIVE_RET if "cycle" and ret_level > N, otherwise continue
537 *
538 */
539
zend_jit_trace_execute(zend_execute_data * ex,const zend_op * op,zend_jit_trace_rec * trace_buffer,uint8_t start,uint32_t is_megamorphic)540 zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphic)
541
542 {
543 #ifdef HAVE_GCC_GLOBAL_REGS
544 zend_execute_data *save_execute_data = execute_data;
545 const zend_op *save_opline = opline;
546 #endif
547 const zend_op *orig_opline, *end_opline;
548 zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
549 zend_jit_trace_stop halt = 0;
550 int level = 0;
551 int ret_level = 0;
552 zend_vm_opcode_handler_t handler;
553 const zend_op_array *op_array;
554 zend_jit_op_array_trace_extension *jit_extension;
555 size_t offset;
556 int idx, count;
557 uint8_t trace_flags, op1_type, op2_type, op3_type;
558 zend_class_entry *ce1, *ce2;
559 const zend_op *link_to_enter_opline = NULL;
560 int backtrack_link_to_enter = -1;
561 int backtrack_recursion = -1;
562 int backtrack_ret_recursion = -1;
563 int backtrack_ret_recursion_level = 0;
564 int loop_unroll_limit = 0;
565 int last_loop = -1;
566 int last_loop_level = -1;
567 const zend_op *last_loop_opline = NULL;
568 const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
569 #ifdef HAVE_GCC_GLOBAL_REGS
570 zend_execute_data *prev_execute_data = ex;
571
572 execute_data = ex;
573 opline = EX(opline) = op;
574 #else
575 int rc;
576 zend_execute_data *execute_data = ex;
577 const zend_op *opline = EX(opline);
578 #endif
579 zend_execute_data *prev_call = EX(call);
580
581 orig_opline = opline;
582
583 op_array = &EX(func)->op_array;
584 jit_extension =
585 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
586 offset = jit_extension->offset;
587 if (!op_array->function_name
588 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
589 op_array = jit_extension->op_array;
590 }
591
592 TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline);
593
594 if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
595 /* Abort trace because of exception */
596 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_EXCEPTION, opline);
597 #ifdef HAVE_GCC_GLOBAL_REGS
598 execute_data = save_execute_data;
599 opline = save_opline;
600 #endif
601 return ZEND_JIT_TRACE_STOP_EXCEPTION;
602 }
603
604 if (prev_call) {
605 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic);
606 if (ret < 0) {
607 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_BAD_FUNC, opline);
608 #ifdef HAVE_GCC_GLOBAL_REGS
609 execute_data = save_execute_data;
610 opline = save_opline;
611 #endif
612 return ZEND_JIT_TRACE_STOP_BAD_FUNC;
613 }
614 idx = ret;
615 }
616
617 while (1) {
618 ce1 = ce2 = NULL;
619 op1_type = op2_type = op3_type = IS_UNKNOWN;
620 if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV))
621 && opline->opcode != ZEND_ROPE_ADD
622 && opline->opcode != ZEND_ROPE_END
623 && opline->opcode != ZEND_NEW
624 && opline->opcode != ZEND_FETCH_CLASS_CONSTANT
625 && opline->opcode != ZEND_INIT_STATIC_METHOD_CALL) {
626 zval *zv = EX_VAR(opline->op1.var);
627 op1_type = Z_TYPE_P(zv);
628 uint8_t flags = 0;
629
630 if (op1_type == IS_INDIRECT) {
631 zv = Z_INDIRECT_P(zv);
632 op1_type = Z_TYPE_P(zv);
633 flags |= IS_TRACE_INDIRECT;
634 }
635 if (op1_type == IS_REFERENCE) {
636 zv = Z_REFVAL_P(zv);
637 op1_type = Z_TYPE_P(zv);
638 flags |= IS_TRACE_REFERENCE;
639 }
640 if (Z_TYPE_P(zv) == IS_OBJECT) {
641 ce1 = Z_OBJCE_P(zv);
642 } else if (Z_TYPE_P(zv) == IS_ARRAY) {
643 if (HT_IS_PACKED(Z_ARRVAL_P(zv))) {
644 flags |= IS_TRACE_PACKED;
645 }
646 }
647 op1_type |= flags;
648 }
649 if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)
650 && opline->opcode != ZEND_INSTANCEOF
651 && opline->opcode != ZEND_UNSET_STATIC_PROP
652 && opline->opcode != ZEND_ISSET_ISEMPTY_STATIC_PROP
653 && opline->opcode != ZEND_ASSIGN_STATIC_PROP
654 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_REF
655 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_OP
656 && opline->opcode != ZEND_PRE_INC_STATIC_PROP
657 && opline->opcode != ZEND_POST_INC_STATIC_PROP
658 && opline->opcode != ZEND_PRE_DEC_STATIC_PROP
659 && opline->opcode != ZEND_POST_DEC_STATIC_PROP
660 && opline->opcode != ZEND_FETCH_STATIC_PROP_R
661 && opline->opcode != ZEND_FETCH_STATIC_PROP_W
662 && opline->opcode != ZEND_FETCH_STATIC_PROP_RW
663 && opline->opcode != ZEND_FETCH_STATIC_PROP_IS
664 && opline->opcode != ZEND_FETCH_STATIC_PROP_FUNC_ARG
665 && opline->opcode != ZEND_FETCH_STATIC_PROP_UNSET
666 && (opline->op2_type == IS_CV
667 || (opline->opcode != ZEND_FE_FETCH_R
668 && opline->opcode != ZEND_FE_FETCH_RW))) {
669 zval *zv = EX_VAR(opline->op2.var);
670 uint8_t flags = 0;
671
672 op2_type = Z_TYPE_P(zv);
673 if (op2_type == IS_INDIRECT) {
674 zv = Z_INDIRECT_P(zv);
675 op2_type = Z_TYPE_P(zv);
676 flags |= IS_TRACE_INDIRECT;
677 }
678 if (op2_type == IS_REFERENCE) {
679 zv = Z_REFVAL_P(zv);
680 op2_type = Z_TYPE_P(zv);
681 flags |= IS_TRACE_REFERENCE;
682 }
683 if (Z_TYPE_P(zv) == IS_OBJECT) {
684 ce2 = Z_OBJCE_P(zv);
685 }
686 op2_type |= flags;
687 }
688 if (opline->opcode == ZEND_ASSIGN_DIM ||
689 opline->opcode == ZEND_ASSIGN_OBJ ||
690 opline->opcode == ZEND_ASSIGN_STATIC_PROP ||
691 opline->opcode == ZEND_ASSIGN_DIM_OP ||
692 opline->opcode == ZEND_ASSIGN_OBJ_OP ||
693 opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP ||
694 opline->opcode == ZEND_ASSIGN_OBJ_REF ||
695 opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF) {
696 if ((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
697 zval *zv = EX_VAR((opline+1)->op1.var);
698 uint8_t flags = 0;
699
700 op3_type = Z_TYPE_P(zv);
701 if (op3_type == IS_INDIRECT) {
702 zv = Z_INDIRECT_P(zv);
703 op3_type = Z_TYPE_P(zv);
704 flags |= IS_TRACE_INDIRECT;
705 }
706 if (op3_type == IS_REFERENCE) {
707 zv = Z_REFVAL_P(zv);
708 op3_type = Z_TYPE_P(zv);
709 flags |= IS_TRACE_REFERENCE;
710 }
711 op3_type |= flags;
712 }
713 }
714
715 TRACE_RECORD_VM(ZEND_JIT_TRACE_VM, opline, op1_type, op2_type, op3_type);
716
717 if (ce1) {
718 TRACE_RECORD(ZEND_JIT_TRACE_OP1_TYPE, 0, ce1);
719 }
720
721 if (ce2) {
722 TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2);
723 }
724
725 if (opline->opcode == ZEND_DO_FCALL
726 || opline->opcode == ZEND_DO_ICALL
727 || opline->opcode == ZEND_DO_UCALL
728 || opline->opcode == ZEND_DO_FCALL_BY_NAME) {
729 if (ZEND_CALL_INFO(EX(call)) & ZEND_CALL_MEGAMORPHIC) {
730 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
731 break;
732 }
733 if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
734 if (EX(call)->func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) {
735 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
736 break;
737 }
738 TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func);
739 }
740 } else if (opline->opcode == ZEND_INCLUDE_OR_EVAL) {
741 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
742 break;
743 }
744
745 handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
746 #ifdef HAVE_GCC_GLOBAL_REGS
747 handler();
748 if (UNEXPECTED(opline == zend_jit_halt_op)) {
749 stop = ZEND_JIT_TRACE_STOP_RETURN;
750 opline = NULL;
751 halt = ZEND_JIT_TRACE_HALT;
752 break;
753 }
754 if (UNEXPECTED(execute_data != prev_execute_data)) {
755 #else
756 rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);
757 if (rc != 0) {
758 if (rc < 0) {
759 stop = ZEND_JIT_TRACE_STOP_RETURN;
760 opline = NULL;
761 halt = ZEND_JIT_TRACE_HALT;
762 break;
763 } else if (execute_data == EG(current_execute_data)) {
764 /* return after interrupt handler */
765 rc = 0;
766 }
767 execute_data = EG(current_execute_data);
768 opline = EX(opline);
769 #endif
770
771 op_array = &EX(func)->op_array;
772 jit_extension =
773 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
774 if (UNEXPECTED(!jit_extension)
775 || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) {
776 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
777 break;
778 }
779 offset = jit_extension->offset;
780 if (!op_array->function_name
781 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
782 op_array = jit_extension->op_array;
783 }
784
785 #ifdef HAVE_GCC_GLOBAL_REGS
786 if (execute_data->prev_execute_data == prev_execute_data) {
787 #else
788 if (rc == 0) {
789 /* pass */
790 } else if (rc == 1) {
791 #endif
792 /* Enter into function */
793 prev_call = NULL;
794 if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) {
795 stop = ZEND_JIT_TRACE_STOP_TOO_DEEP;
796 break;
797 }
798
799 if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
800 /* TODO: Can we continue recording ??? */
801 stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
802 break;
803 }
804
805 TRACE_RECORD(ZEND_JIT_TRACE_ENTER,
806 EX(return_value) != NULL ? ZEND_JIT_TRACE_RETRUN_VALUE_USED : 0,
807 op_array);
808
809 count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level);
810
811 if (opline == orig_opline) {
812 if (count + 1 >= JIT_G(max_recursive_calls)) {
813 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
814 break;
815 }
816 backtrack_recursion = idx;
817 } else if (count >= JIT_G(max_recursive_calls)) {
818 stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
819 break;
820 }
821
822 unrolled_calls[ret_level + level] = &EX(func)->op_array;
823 level++;
824 } else {
825 /* Return from function */
826 prev_call = EX(call);
827 if (level == 0) {
828 if (start == ZEND_JIT_TRACE_START_RETURN
829 && JIT_G(max_recursive_returns) > 0
830 && execute_data->prev_execute_data
831 && execute_data->prev_execute_data->func
832 && execute_data->prev_execute_data->func->type == ZEND_USER_FUNCTION
833 && zend_jit_trace_has_recursive_ret(execute_data, trace_buffer[0].op_array, orig_opline, ret_level)) {
834 if (ret_level > ZEND_JIT_TRACE_MAX_RET_DEPTH) {
835 stop = ZEND_JIT_TRACE_STOP_TOO_DEEP_RET;
836 break;
837 }
838 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
839 count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level);
840 if (opline == orig_opline) {
841 if (count + 1 >= JIT_G(max_recursive_returns)) {
842 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
843 break;
844 }
845 backtrack_ret_recursion = idx;
846 backtrack_ret_recursion_level = ret_level;
847 } else if (count >= JIT_G(max_recursive_returns)) {
848 stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
849 break;
850 }
851
852 unrolled_calls[ret_level] = &EX(func)->op_array;
853 ret_level++;
854 last_loop_opline = NULL;
855
856 if (prev_call) {
857 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0);
858 if (ret < 0) {
859 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
860 break;
861 }
862 idx = ret;
863 }
864 } else if (start & ZEND_JIT_TRACE_START_LOOP
865 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
866 ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
867 /* Fail to try close the loop.
868 If this doesn't work terminate it. */
869 stop = ZEND_JIT_TRACE_STOP_LOOP_EXIT;
870 break;
871 } else if (start & ZEND_JIT_TRACE_START_ENTER
872 && EX(prev_execute_data)
873 && EX(func) == EX(prev_execute_data)->func
874 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
875 ZEND_JIT_TRACE_STOP_RECURSION_EXIT) {
876 stop = ZEND_JIT_TRACE_STOP_RECURSION_EXIT;
877 break;
878 } else {
879 stop = ZEND_JIT_TRACE_STOP_RETURN;
880 break;
881 }
882 } else {
883 level--;
884 if (level < last_loop_level) {
885 last_loop_opline = NULL;
886 }
887 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
888 }
889 }
890 #ifdef HAVE_GCC_GLOBAL_REGS
891 prev_execute_data = execute_data;
892 #endif
893 }
894 if (EX(call) != prev_call) {
895 if (EX(call)
896 && EX(call)->prev_execute_data == prev_call) {
897 zend_function *func;
898 zend_jit_op_array_trace_extension *jit_extension;
899
900 if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
901 /* TODO: Can we continue recording ??? */
902 stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
903 break;
904 } else if (EX(call)->func->common.fn_flags & ZEND_ACC_NEVER_CACHE) {
905 /* TODO: Can we continue recording ??? */
906 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
907 break;
908 }
909 func = EX(call)->func;
910 if (func->type == ZEND_INTERNAL_FUNCTION
911 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
912 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
913 break;
914 }
915 if (func->type == ZEND_USER_FUNCTION
916 && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
917 jit_extension =
918 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
919 if (UNEXPECTED(!jit_extension)
920 || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)
921 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) {
922 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
923 break;
924 }
925 func = (zend_function*)jit_extension->op_array;
926 }
927
928 #ifndef HAVE_GCC_GLOBAL_REGS
929 opline = EX(opline);
930 #endif
931
932 if (JIT_G(max_polymorphic_calls) == 0
933 && zend_jit_may_be_polymorphic_call(opline - 1)) {
934 func = NULL;
935 } else if ((is_megamorphic == ZEND_JIT_EXIT_METHOD_CALL
936 || is_megamorphic == ZEND_JIT_EXIT_CLOSURE_CALL)
937 && trace_buffer[1].opline == opline - 1) {
938 func = NULL;
939 }
940 if (!func) {
941 ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC);
942 }
943 TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func);
944 }
945 prev_call = EX(call);
946 }
947
948 #ifndef HAVE_GCC_GLOBAL_REGS
949 opline = EX(opline);
950 #endif
951
952 if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
953 /* Abort trace because of exception */
954 stop = ZEND_JIT_TRACE_STOP_EXCEPTION;
955 break;
956 }
957
958 trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
959 if (trace_flags) {
960 if (trace_flags & ZEND_JIT_TRACE_JITED) {
961 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
962 if ((start & ZEND_JIT_TRACE_START_LOOP) != 0
963 && level + ret_level == 0
964 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
965 ZEND_JIT_TRACE_STOP_COMPILED_LOOP) {
966 /* Fail to try close outer loop through side exit.
967 If this doesn't work just link. */
968 stop = ZEND_JIT_TRACE_STOP_COMPILED_LOOP;
969 break;
970 } else {
971 stop = ZEND_JIT_TRACE_STOP_LINK;
972 break;
973 }
974 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
975 if (start != ZEND_JIT_TRACE_START_RETURN) {
976 // TODO: We may try to inline function ???
977 stop = ZEND_JIT_TRACE_STOP_LINK;
978 break;
979 }
980 if (backtrack_link_to_enter < 0) {
981 backtrack_link_to_enter = idx;
982 link_to_enter_opline = opline;
983 }
984 } else {
985 stop = ZEND_JIT_TRACE_STOP_LINK;
986 break;
987 }
988 } else if (trace_flags & ZEND_JIT_TRACE_BLACKLISTED) {
989 stop = ZEND_JIT_TRACE_STOP_BLACK_LIST;
990 break;
991 } else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
992 uint8_t bad_stop;
993
994 if (start != ZEND_JIT_TRACE_START_SIDE) {
995 if (opline == orig_opline && level + ret_level == 0) {
996 stop = ZEND_JIT_TRACE_STOP_LOOP;
997 break;
998 }
999 }
1000
1001 if (start != ZEND_JIT_TRACE_START_SIDE
1002 || level + ret_level != 0) {
1003 /* First try creating a trace for inner loop.
1004 If this doesn't work try loop unroling. */
1005 bad_stop = zend_jit_trace_bad_stop_event(opline,
1006 JIT_G(blacklist_root_trace) / 2);
1007 if (bad_stop != ZEND_JIT_TRACE_STOP_INNER_LOOP
1008 && bad_stop != ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
1009 if (start == ZEND_JIT_TRACE_START_SIDE
1010 || zend_jit_trace_bad_stop_event(orig_opline,
1011 JIT_G(blacklist_root_trace) / 2) != ZEND_JIT_TRACE_STOP_INNER_LOOP) {
1012 stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
1013 break;
1014 }
1015 }
1016 }
1017
1018 if (opline == last_loop_opline
1019 && level == last_loop_level) {
1020 idx = zend_jit_trace_subtrace(trace_buffer,
1021 last_loop, idx, ZEND_JIT_TRACE_START_LOOP, op_array, opline);
1022 start = ZEND_JIT_TRACE_START_LOOP;
1023 stop = ZEND_JIT_TRACE_STOP_LOOP;
1024 ret_level = 0;
1025 break;
1026 } else if (loop_unroll_limit < JIT_G(max_loop_unrolls)) {
1027 last_loop = idx;
1028 last_loop_opline = opline;
1029 last_loop_level = level;
1030 loop_unroll_limit++;
1031 } else {
1032 stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL;
1033 break;
1034 }
1035 } else if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
1036 TRACE_RECORD(ZEND_JIT_TRACE_VM, 0, opline);
1037 stop = ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
1038 break;
1039 }
1040 }
1041 }
1042
1043 end_opline = opline;
1044 if (!ZEND_JIT_TRACE_STOP_OK(stop)) {
1045 if (backtrack_recursion > 0) {
1046 idx = backtrack_recursion;
1047 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
1048 end_opline = orig_opline;
1049 } else if (backtrack_ret_recursion > 0) {
1050 idx = backtrack_ret_recursion;
1051 ret_level = backtrack_ret_recursion_level;
1052 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
1053 end_opline = orig_opline;
1054 } else if (backtrack_link_to_enter > 0) {
1055 if (stop == ZEND_JIT_TRACE_STOP_DEEP_RECURSION
1056 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) / 2) ==
1057 ZEND_JIT_TRACE_STOP_DEEP_RECURSION) {
1058 idx = backtrack_link_to_enter;
1059 stop = ZEND_JIT_TRACE_STOP_LINK;
1060 end_opline = link_to_enter_opline;
1061 }
1062 }
1063 }
1064
1065 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
1066 /* Shrink fake INIT_CALLs */
1067 while (trace_buffer[idx-1].op == ZEND_JIT_TRACE_INIT_CALL
1068 && (trace_buffer[idx-1].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1069 idx--;
1070 }
1071 }
1072
1073 TRACE_END(ZEND_JIT_TRACE_END, stop, end_opline);
1074
1075 #ifdef HAVE_GCC_GLOBAL_REGS
1076 if (!halt) {
1077 EX(opline) = opline;
1078 }
1079 #endif
1080
1081 #ifdef HAVE_GCC_GLOBAL_REGS
1082 execute_data = save_execute_data;
1083 opline = save_opline;
1084 #endif
1085
1086 return stop | halt;
1087 }
1088
