1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 | Xinchen Hui <laruence@php.net> |
17 +----------------------------------------------------------------------+
18 */
19
20 #include "Zend/zend_execute.h"
21 #include "Zend/zend_exceptions.h"
22 #include "Zend/zend_vm.h"
23 #include "Zend/zend_closures.h"
24 #include "Zend/zend_constants.h"
25 #include "Zend/zend_API.h"
26
27 #include <ZendAccelerator.h>
28 #include "Optimizer/zend_func_info.h"
29 #include "Optimizer/zend_call_graph.h"
30 #include "zend_jit.h"
31 #if ZEND_JIT_TARGET_X86
32 # include "zend_jit_x86.h"
33 #elif ZEND_JIT_TARGET_ARM64
34 # include "zend_jit_arm64.h"
35 #endif
36
37 #include "zend_jit_internal.h"
38
39 #ifdef HAVE_GCC_GLOBAL_REGS
40 # pragma GCC diagnostic ignored "-Wvolatile-register-var"
41 # if defined(__x86_64__)
42 register zend_execute_data* volatile execute_data __asm__("%r14");
43 register const zend_op* volatile opline __asm__("%r15");
44 # elif defined(i386)
45 register zend_execute_data* volatile execute_data __asm__("%esi");
46 register const zend_op* volatile opline __asm__("%edi");
47 # elif defined(__aarch64__)
48 register zend_execute_data* volatile execute_data __asm__("x27");
49 register const zend_op* volatile opline __asm__("x28");
50 # endif
51 # pragma GCC diagnostic warning "-Wvolatile-register-var"
52 #endif
53
zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)54 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)
55 {
56 zend_execute_data *old_execute_data;
57
58 if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
59 zend_clean_and_cache_symbol_table(EX(symbol_table));
60 }
61
62 zend_vm_stack_free_extra_args_ex(call_info, execute_data);
63 if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) {
64 OBJ_RELEASE(Z_OBJ(execute_data->This));
65 } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
66 OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
67 }
68 if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
69 zend_free_extra_named_params(EX(extra_named_params));
70 }
71
72 old_execute_data = execute_data;
73 execute_data = EX(prev_execute_data);
74 zend_vm_stack_free_call_frame_ex(call_info, old_execute_data);
75
76 if (UNEXPECTED(EG(exception) != NULL)) {
77 const zend_op *old_opline = EX(opline);
78 zend_throw_exception_internal(NULL);
79 if (old_opline->result_type != IS_UNDEF) {
80 zval_ptr_dtor(EX_VAR(old_opline->result.var));
81 }
82 #ifndef HAVE_GCC_GLOBAL_REGS
83 return 2; // ZEND_VM_LEAVE
84 #endif
85 } else {
86 EX(opline)++;
87 #ifdef HAVE_GCC_GLOBAL_REGS
88 opline = EX(opline);
89 #else
90 return 2; // ZEND_VM_LEAVE
91 #endif
92 }
93 }
94
zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)95 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)
96 {
97 if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) {
98 if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
99 zend_clean_and_cache_symbol_table(EX(symbol_table));
100 }
101 zend_vm_stack_free_extra_args_ex(call_info, execute_data);
102 }
103 if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
104 zend_free_extra_named_params(EX(extra_named_params));
105 }
106 if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
107 OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
108 }
109 execute_data = EG(current_execute_data);
110 #ifdef HAVE_GCC_GLOBAL_REGS
111 opline = zend_jit_halt_op;
112 #else
113 return -1; // ZEND_VM_RETURN
114 #endif
115 }
116
zend_jit_leave_func_helper(EXECUTE_DATA_D)117 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(EXECUTE_DATA_D)
118 {
119 uint32_t call_info = EX_CALL_INFO();
120
121 if (call_info & ZEND_CALL_TOP) {
122 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info);
123 } else {
124 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info);
125 }
126 }
127
zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)128 void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)
129 {
130 zend_op_array *op_array = &EX(func)->op_array;
131
132 if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) {
133 uint32_t first_extra_arg = op_array->num_args;
134 uint32_t num_args = EX_NUM_ARGS();
135 zval *end, *src, *dst;
136 uint32_t type_flags = 0;
137
138 if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) {
139 /* Skip useless ZEND_RECV and ZEND_RECV_INIT opcodes */
140 #ifdef HAVE_GCC_GLOBAL_REGS
141 opline += first_extra_arg;
142 #else
143 EX(opline) += first_extra_arg;
144 #endif
145 }
146
147 /* move extra args into separate array after all CV and TMP vars */
148 end = EX_VAR_NUM(first_extra_arg - 1);
149 src = end + (num_args - first_extra_arg);
150 dst = src + (op_array->last_var + op_array->T - first_extra_arg);
151 if (EXPECTED(src != dst)) {
152 do {
153 type_flags |= Z_TYPE_INFO_P(src);
154 ZVAL_COPY_VALUE(dst, src);
155 ZVAL_UNDEF(src);
156 src--;
157 dst--;
158 } while (src != end);
159 if (type_flags & (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT)) {
160 ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
161 }
162 } else {
163 do {
164 if (Z_REFCOUNTED_P(src)) {
165 ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
166 break;
167 }
168 src--;
169 } while (src != end);
170 }
171 }
172 }
173
zend_jit_deprecated_helper(OPLINE_D)174 bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D)
175 {
176 zend_execute_data *call = (zend_execute_data *) opline;
177 zend_function *fbc = call->func;
178
179 zend_deprecated_function(fbc);
180
181 if (EG(exception)) {
182 #ifndef HAVE_GCC_GLOBAL_REGS
183 zend_execute_data *execute_data = EG(current_execute_data);
184 #endif
185 const zend_op *opline = EG(opline_before_exception);
186 if (RETURN_VALUE_USED(opline)) {
187 ZVAL_UNDEF(EX_VAR(opline->result.var));
188 }
189
190 zend_vm_stack_free_args(call);
191
192 if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) {
193 OBJ_RELEASE(Z_OBJ(call->This));
194 }
195
196 zend_vm_stack_free_call_frame(call);
197 return 0;
198 }
199 return 1;
200 }
201
zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)202 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)
203 {
204 zend_op_array *op_array = (zend_op_array*)EX(func);
205 zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
206 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t) jit_extension->orig_handler;
207 ++*(uintptr_t*)(EX(run_time_cache) + zend_jit_profile_counter_rid);
208 ++zend_jit_profile_counter;
209 ZEND_OPCODE_TAIL_CALL(handler);
210 }
211
zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)212 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
213 {
214 zend_jit_op_array_hot_extension *jit_extension =
215 (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
216 #ifndef HAVE_GCC_GLOBAL_REGS
217 const zend_op *opline = EX(opline);
218 #endif
219
220 *(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func));
221
222 if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
223 *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
224 zend_jit_hot_func(execute_data, opline);
225 ZEND_OPCODE_RETURN();
226 } else {
227 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
228 ZEND_OPCODE_TAIL_CALL(handler);
229 }
230 }
231
zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)232 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
233 {
234 zend_jit_op_array_hot_extension *jit_extension =
235 (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
236 #ifndef HAVE_GCC_GLOBAL_REGS
237 const zend_op *opline = EX(opline);
238 #endif
239
240 *(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop));
241
242 if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
243 *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
244 zend_jit_hot_func(execute_data, opline);
245 ZEND_OPCODE_RETURN();
246 } else {
247 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
248 ZEND_OPCODE_TAIL_CALL(handler);
249 }
250 }
251
_zend_quick_get_constant(const zval * key,uint32_t flags,int check_defined_only)252 static zend_always_inline zend_constant* _zend_quick_get_constant(
253 const zval *key, uint32_t flags, int check_defined_only)
254 {
255 #ifndef HAVE_GCC_GLOBAL_REGS
256 zend_execute_data *execute_data = EG(current_execute_data);
257 #endif
258 const zend_op *opline = EX(opline);
259 zval *zv;
260 zend_constant *c = NULL;
261
262 /* null/true/false are resolved during compilation, so don't check for them here. */
263 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
264 if (zv) {
265 c = (zend_constant*)Z_PTR_P(zv);
266 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
267 key++;
268 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
269 if (zv) {
270 c = (zend_constant*)Z_PTR_P(zv);
271 }
272 }
273
274 if (!c) {
275 if (!check_defined_only) {
276 zend_throw_error(NULL, "Undefined constant \"%s\"", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)));
277 ZVAL_UNDEF(EX_VAR(opline->result.var));
278 }
279 CACHE_PTR(opline->extended_value, ENCODE_SPECIAL_CACHE_NUM(zend_hash_num_elements(EG(zend_constants))));
280 return NULL;
281 }
282
283 if (!check_defined_only) {
284 if (ZEND_CONSTANT_FLAGS(c) & CONST_DEPRECATED) {
285 zend_error(E_DEPRECATED, "Constant %s is deprecated", ZSTR_VAL(c->name));
286 if (EG(exception)) {
287 return NULL;
288 }
289 return c;
290 }
291 }
292
293 CACHE_PTR(opline->extended_value, c);
294 return c;
295 }
296
zend_jit_get_constant(const zval * key,uint32_t flags)297 zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags)
298 {
299 return _zend_quick_get_constant(key, flags, 0);
300 }
301
zend_jit_check_constant(const zval * key)302 zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key)
303 {
304 return _zend_quick_get_constant(key, 0, 1);
305 }
306
zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)307 static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)
308 {
309 zend_jit_op_array_trace_extension *jit_extension =
310 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
311 size_t offset = jit_extension->offset;
312 #ifndef HAVE_GCC_GLOBAL_REGS
313 const zend_op *opline = EX(opline);
314 #endif
315
316 *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
317
318 if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
319 *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT;
320 if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
321 #ifdef HAVE_GCC_GLOBAL_REGS
322 opline = NULL;
323 return;
324 #else
325 return -1;
326 #endif
327 }
328 #ifdef HAVE_GCC_GLOBAL_REGS
329 execute_data = EG(current_execute_data);
330 opline = execute_data ? EX(opline) : NULL;
331 return;
332 #else
333 return 1;
334 #endif
335 } else {
336 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
337 ZEND_OPCODE_TAIL_CALL(handler);
338 }
339 }
340
zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)341 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
342 {
343 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
344 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
345 }
346
zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)347 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
348 {
349 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
350 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
351 }
352
zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)353 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
354 {
355 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
356 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
357 }
358
359 #define TRACE_RECORD(_op, _info, _ptr) \
360 trace_buffer[idx].info = _op | (_info); \
361 trace_buffer[idx].ptr = _ptr; \
362 idx++; \
363 if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 2) { \
364 stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
365 break; \
366 }
367
368 #define TRACE_RECORD_VM(_op, _ptr, _op1_type, _op2_type, _op3_type) \
369 trace_buffer[idx].op = _op; \
370 trace_buffer[idx].op1_type = _op1_type; \
371 trace_buffer[idx].op2_type = _op2_type; \
372 trace_buffer[idx].op3_type = _op3_type; \
373 trace_buffer[idx].ptr = _ptr; \
374 idx++; \
375 if (idx >= ZEND_JIT_TRACE_MAX_LENGTH - 2) { \
376 stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
377 break; \
378 }
379
380 #define TRACE_START(_op, _start, _ptr1, _ptr2) \
381 trace_buffer[0].op = _op; \
382 trace_buffer[0].start = _start; \
383 trace_buffer[0].level = 0; \
384 trace_buffer[0].ptr = _ptr1; \
385 trace_buffer[1].last = 0; \
386 trace_buffer[1].ptr = _ptr2; \
387 idx = ZEND_JIT_TRACE_START_REC_SIZE;
388
389 #define TRACE_END(_op, _stop, _ptr) \
390 trace_buffer[1].last = idx; \
391 trace_buffer[idx].op = _op; \
392 trace_buffer[idx].start = trace_buffer[idx].start; \
393 trace_buffer[idx].stop = trace_buffer[0].stop = _stop; \
394 trace_buffer[idx].level = trace_buffer[0].level = ret_level ? ret_level + 1 : 0; \
395 trace_buffer[idx].ptr = _ptr;
396
zend_jit_trace_recursive_call_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level,int level)397 static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level)
398 {
399 int i;
400 int count = 0;
401
402 for (i = ret_level; i < level; i++) {
403 count += (unrolled_calls[i] == op_array);
404 }
405 return count;
406 }
407
zend_jit_trace_recursive_ret_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level)408 static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level)
409 {
410 int i;
411 int count = 0;
412
413 for (i = 0; i < ret_level; i++) {
414 count += (unrolled_calls[i] == op_array);
415 }
416 return count;
417 }
418
zend_jit_trace_has_recursive_ret(zend_execute_data * ex,const zend_op_array * orig_op_array,const zend_op * orig_opline,int ret_level)419 static int zend_jit_trace_has_recursive_ret(zend_execute_data *ex, const zend_op_array *orig_op_array, const zend_op *orig_opline, int ret_level)
420 {
421 while (ex != NULL && ex->func != NULL && ret_level < ZEND_JIT_TRACE_MAX_RET_DEPTH) {
422 if (&ex->func->op_array == orig_op_array && ex->opline + 1 == orig_opline) {
423 return 1;
424 }
425 ex = ex->prev_execute_data;
426 ret_level++;
427 }
428 return 0;
429 }
430
zend_jit_trace_bad_stop_event(const zend_op * opline,int count)431 static uint8_t zend_jit_trace_bad_stop_event(const zend_op *opline, int count)
432 {
433 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
434 uint8_t *cache_count = JIT_G(bad_root_cache_count);
435 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
436 uint32_t i;
437
438 if (count < 0) {
439 count = 0;
440 }
441 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
442 if (cache_opline[i] == opline) {
443 if (cache_count[i] >= count) {
444 return cache_stop[i];
445 }
446 break;
447 }
448 }
449 return 0;
450 }
451
452 #define ZEND_CALL_MEGAMORPHIC ZEND_CALL_JIT_RESERVED
453
zend_jit_trace_record_fake_init_call_ex(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic,uint32_t init_level)454 static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t init_level)
455 {
456 zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
457
458 do {
459 zend_function *func;
460 zend_jit_op_array_trace_extension *jit_extension;
461
462 if (call->prev_execute_data) {
463 idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, init_level + 1);
464 if (idx < 0) {
465 return idx;
466 }
467 }
468
469 func = call->func;
470 if (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE|ZEND_ACC_NEVER_CACHE)) {
471 /* TODO: Can we continue recording ??? */
472 return -1;
473 }
474 if (func->type == ZEND_INTERNAL_FUNCTION
475 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
476 return -1;
477 }
478 if (func->type == ZEND_USER_FUNCTION
479 && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
480 jit_extension =
481 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
482 if (UNEXPECTED(!jit_extension
483 || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)
484 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE))) {
485 return -1;
486 }
487 func = (zend_function*)jit_extension->op_array;
488 }
489 if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM
490 /* TODO: use more accurate check ??? */
491 && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
492 || func->common.scope)) {
493 func = NULL;
494 ZEND_ADD_CALL_FLAG(call, ZEND_CALL_MEGAMORPHIC);
495 }
496 TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INFO(init_level), func);
497 } while (0);
498 return idx;
499 }
500
zend_jit_trace_record_fake_init_call(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic)501 static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic)
502 {
503 return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, 0);
504 }
505
zend_jit_trace_subtrace(zend_jit_trace_rec * trace_buffer,int start,int end,uint8_t event,const zend_op_array * op_array,const zend_op * opline)506 static int zend_jit_trace_subtrace(zend_jit_trace_rec *trace_buffer, int start, int end, uint8_t event, const zend_op_array *op_array, const zend_op *opline)
507 {
508 int idx;
509
510 TRACE_START(ZEND_JIT_TRACE_START, event, op_array, opline);
511 memmove(trace_buffer + idx, trace_buffer + start, (end - start) * sizeof(zend_jit_trace_rec));
512 return idx + (end - start);
513 }
514
515 /*
516 * Trace Linking Rules
517 * ===================
518 *
519 * flags
520 * +----------+----------+----------++----------+----------+----------+
521 * | || JIT |
522 * +----------+----------+----------++----------+----------+----------+
523 * start | LOOP | ENTER | RETURN || LOOP | ENTER | RETURN |
524 * +========+==========+==========+==========++==========+==========+==========+
525 * | LOOP | loop | | loop-ret || COMPILED | LINK | LINK |
526 * +--------+----------+----------+----------++----------+----------+----------+
527 * | ENTER |INNER_LOOP| rec-call | return || LINK | LINK | LINK |
528 * +--------+----------+----------+----------++----------+----------+----------+
529 * | RETURN |INNER_LOOP| | rec-ret || LINK | | LINK |
530 * +--------+----------+----------+----------++----------+----------+----------+
531 * | SIDE | unroll | | return || LINK | LINK | LINK |
532 * +--------+----------+----------+----------++----------+----------+----------+
533 *
534 * loop: LOOP if "cycle" and level == 0, otherwise INNER_LOOP
535 * INNER_LOOP: abort recording and start new one (wait for loop)
536 * COMPILED: abort recording (wait while side exit creates outer loop)
537 * unroll: continue recording while unroll limit reached
538 * rec-call: RECURSIVE_CALL if "cycle" and level > N, otherwise continue
539 * loop-ret: LOOP_EXIT if level == 0, otherwise continue (wait for loop)
540 * return: RETURN if level == 0
541 * rec_ret: RECURSIVE_RET if "cycle" and ret_level > N, otherwise continue
542 *
543 */
544
zend_jit_trace_execute(zend_execute_data * ex,const zend_op * op,zend_jit_trace_rec * trace_buffer,uint8_t start,uint32_t is_megamorphic)545 zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphic)
546
547 {
548 #ifdef HAVE_GCC_GLOBAL_REGS
549 zend_execute_data *save_execute_data = execute_data;
550 const zend_op *save_opline = opline;
551 #endif
552 const zend_op *orig_opline, *end_opline;
553 zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
554 zend_jit_trace_stop halt = 0;
555 int level = 0;
556 int ret_level = 0;
557 zend_vm_opcode_handler_t handler;
558 const zend_op_array *op_array;
559 zend_jit_op_array_trace_extension *jit_extension;
560 size_t offset;
561 int idx, count;
562 uint8_t trace_flags, op1_type, op2_type, op3_type;
563 zend_class_entry *ce1, *ce2;
564 const zend_op *link_to_enter_opline = NULL;
565 int backtrack_link_to_enter = -1;
566 int backtrack_recursion = -1;
567 int backtrack_ret_recursion = -1;
568 int backtrack_ret_recursion_level = 0;
569 int loop_unroll_limit = 0;
570 int last_loop = -1;
571 int last_loop_level = -1;
572 const zend_op *last_loop_opline = NULL;
573 const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
574 #ifdef HAVE_GCC_GLOBAL_REGS
575 zend_execute_data *prev_execute_data = ex;
576
577 execute_data = ex;
578 opline = EX(opline) = op;
579 #else
580 int rc;
581 zend_execute_data *execute_data = ex;
582 const zend_op *opline = EX(opline);
583 #endif
584 zend_execute_data *prev_call = EX(call);
585
586 orig_opline = opline;
587
588 op_array = &EX(func)->op_array;
589 jit_extension =
590 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
591 offset = jit_extension->offset;
592 if (!op_array->function_name
593 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
594 op_array = jit_extension->op_array;
595 }
596
597 TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline);
598
599 if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
600 /* Abort trace because of exception */
601 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_EXCEPTION, opline);
602 #ifdef HAVE_GCC_GLOBAL_REGS
603 execute_data = save_execute_data;
604 opline = save_opline;
605 #endif
606 return ZEND_JIT_TRACE_STOP_EXCEPTION;
607 }
608
609 if (prev_call) {
610 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic);
611 if (ret < 0) {
612 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_BAD_FUNC, opline);
613 #ifdef HAVE_GCC_GLOBAL_REGS
614 execute_data = save_execute_data;
615 opline = save_opline;
616 #endif
617 return ZEND_JIT_TRACE_STOP_BAD_FUNC;
618 }
619 idx = ret;
620 }
621
622 while (1) {
623 ce1 = ce2 = NULL;
624 op1_type = op2_type = op3_type = IS_UNKNOWN;
625 if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV))
626 && opline->opcode != ZEND_ROPE_ADD
627 && opline->opcode != ZEND_ROPE_END
628 && opline->opcode != ZEND_NEW
629 && opline->opcode != ZEND_FETCH_CLASS_CONSTANT
630 && opline->opcode != ZEND_INIT_STATIC_METHOD_CALL) {
631 zval *zv = EX_VAR(opline->op1.var);
632 op1_type = Z_TYPE_P(zv);
633 uint8_t flags = 0;
634
635 if (op1_type == IS_INDIRECT) {
636 zv = Z_INDIRECT_P(zv);
637 op1_type = Z_TYPE_P(zv);
638 flags |= IS_TRACE_INDIRECT;
639 }
640 if (op1_type == IS_REFERENCE) {
641 zv = Z_REFVAL_P(zv);
642 op1_type = Z_TYPE_P(zv);
643 flags |= IS_TRACE_REFERENCE;
644 }
645 if (Z_TYPE_P(zv) == IS_OBJECT) {
646 ce1 = Z_OBJCE_P(zv);
647 } else if (Z_TYPE_P(zv) == IS_ARRAY) {
648 if (HT_IS_PACKED(Z_ARRVAL_P(zv))) {
649 flags |= IS_TRACE_PACKED;
650 }
651 }
652 op1_type |= flags;
653 }
654 if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)
655 && opline->opcode != ZEND_INSTANCEOF
656 && opline->opcode != ZEND_UNSET_STATIC_PROP
657 && opline->opcode != ZEND_ISSET_ISEMPTY_STATIC_PROP
658 && opline->opcode != ZEND_ASSIGN_STATIC_PROP
659 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_REF
660 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_OP
661 && opline->opcode != ZEND_PRE_INC_STATIC_PROP
662 && opline->opcode != ZEND_POST_INC_STATIC_PROP
663 && opline->opcode != ZEND_PRE_DEC_STATIC_PROP
664 && opline->opcode != ZEND_POST_DEC_STATIC_PROP
665 && opline->opcode != ZEND_FETCH_STATIC_PROP_R
666 && opline->opcode != ZEND_FETCH_STATIC_PROP_W
667 && opline->opcode != ZEND_FETCH_STATIC_PROP_RW
668 && opline->opcode != ZEND_FETCH_STATIC_PROP_IS
669 && opline->opcode != ZEND_FETCH_STATIC_PROP_FUNC_ARG
670 && opline->opcode != ZEND_FETCH_STATIC_PROP_UNSET
671 && (opline->op2_type == IS_CV
672 || (opline->opcode != ZEND_FE_FETCH_R
673 && opline->opcode != ZEND_FE_FETCH_RW))) {
674 zval *zv = EX_VAR(opline->op2.var);
675 uint8_t flags = 0;
676
677 op2_type = Z_TYPE_P(zv);
678 if (op2_type == IS_INDIRECT) {
679 zv = Z_INDIRECT_P(zv);
680 op2_type = Z_TYPE_P(zv);
681 flags |= IS_TRACE_INDIRECT;
682 }
683 if (op2_type == IS_REFERENCE) {
684 zv = Z_REFVAL_P(zv);
685 op2_type = Z_TYPE_P(zv);
686 flags |= IS_TRACE_REFERENCE;
687 }
688 if (Z_TYPE_P(zv) == IS_OBJECT) {
689 ce2 = Z_OBJCE_P(zv);
690 }
691 op2_type |= flags;
692 }
693 if (opline->opcode == ZEND_ASSIGN_DIM ||
694 opline->opcode == ZEND_ASSIGN_OBJ ||
695 opline->opcode == ZEND_ASSIGN_STATIC_PROP ||
696 opline->opcode == ZEND_ASSIGN_DIM_OP ||
697 opline->opcode == ZEND_ASSIGN_OBJ_OP ||
698 opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP ||
699 opline->opcode == ZEND_ASSIGN_OBJ_REF ||
700 opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF) {
701 if ((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
702 zval *zv = EX_VAR((opline+1)->op1.var);
703 uint8_t flags = 0;
704
705 op3_type = Z_TYPE_P(zv);
706 if (op3_type == IS_INDIRECT) {
707 zv = Z_INDIRECT_P(zv);
708 op3_type = Z_TYPE_P(zv);
709 flags |= IS_TRACE_INDIRECT;
710 }
711 if (op3_type == IS_REFERENCE) {
712 zv = Z_REFVAL_P(zv);
713 op3_type = Z_TYPE_P(zv);
714 flags |= IS_TRACE_REFERENCE;
715 }
716 op3_type |= flags;
717 }
718 }
719
720 TRACE_RECORD_VM(ZEND_JIT_TRACE_VM, opline, op1_type, op2_type, op3_type);
721
722 if (ce1) {
723 TRACE_RECORD(ZEND_JIT_TRACE_OP1_TYPE, 0, ce1);
724 }
725
726 if (ce2) {
727 TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2);
728 }
729
730 switch (opline->opcode) {
731 case ZEND_FETCH_DIM_R:
732 case ZEND_FETCH_DIM_W:
733 case ZEND_FETCH_DIM_RW:
734 case ZEND_FETCH_DIM_IS:
735 case ZEND_FETCH_DIM_FUNC_ARG:
736 case ZEND_FETCH_DIM_UNSET:
737 case ZEND_FETCH_LIST_R:
738 case ZEND_FETCH_LIST_W:
739 case ZEND_ASSIGN_DIM:
740 case ZEND_ASSIGN_DIM_OP:
741 case ZEND_UNSET_DIM:
742 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
743 if (opline->op1_type == IS_CONST) {
744 zval *arr = RT_CONSTANT(opline, opline->op1);
745 op1_type = Z_TYPE_P(arr);
746 }
747 if ((op1_type & IS_TRACE_TYPE_MASK) == IS_ARRAY
748 && opline->op2_type != IS_UNDEF) {
749 zval *arr, *dim, *val;
750 uint8_t val_type = IS_UNDEF;
751
752 if (opline->op2_type == IS_CONST) {
753 dim = RT_CONSTANT(opline, opline->op2);
754 } else {
755 dim = EX_VAR(opline->op2.var);
756 }
757
758 if (Z_TYPE_P(dim) == IS_LONG || Z_TYPE_P(dim) == IS_STRING) {
759 if (opline->op1_type == IS_CONST) {
760 arr = RT_CONSTANT(opline, opline->op1);
761 } else {
762 arr = EX_VAR(opline->op1.var);
763 }
764 if (Z_TYPE_P(arr) == IS_INDIRECT) {
765 arr = Z_INDIRECT_P(arr);
766 }
767 if (Z_TYPE_P(arr) == IS_REFERENCE) {
768 arr = Z_REFVAL_P(arr);
769 }
770 ZEND_ASSERT(Z_TYPE_P(arr) == IS_ARRAY);
771 if (Z_TYPE_P(dim) == IS_LONG) {
772 val = zend_hash_index_find(Z_ARRVAL_P(arr), Z_LVAL_P(dim));
773 } else /*if Z_TYPE_P(dim) == IS_STRING)*/ {
774 val = zend_symtable_find(Z_ARRVAL_P(arr), Z_STR_P(dim));
775 }
776 if (val) {
777 val_type = Z_TYPE_P(val);
778 }
779 TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, val_type, 0, 0);
780 }
781 }
782 break;
783 case ZEND_FETCH_OBJ_R:
784 case ZEND_FETCH_OBJ_W:
785 case ZEND_FETCH_OBJ_RW:
786 case ZEND_FETCH_OBJ_IS:
787 case ZEND_FETCH_OBJ_FUNC_ARG:
788 case ZEND_FETCH_OBJ_UNSET:
789 case ZEND_ASSIGN_OBJ:
790 case ZEND_ASSIGN_OBJ_OP:
791 case ZEND_ASSIGN_OBJ_REF:
792 case ZEND_UNSET_OBJ:
793 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
794 case ZEND_PRE_INC_OBJ:
795 case ZEND_PRE_DEC_OBJ:
796 case ZEND_POST_INC_OBJ:
797 case ZEND_POST_DEC_OBJ:
798 if (opline->op1_type != IS_CONST
799 && opline->op2_type == IS_CONST
800 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
801 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
802 zval *obj, *val;
803 zend_string *prop_name = Z_STR_P(RT_CONSTANT(opline, opline->op2));
804 zend_property_info *prop_info;
805
806 if (opline->op1_type == IS_UNUSED) {
807 obj = &EX(This);
808 } else {
809 obj = EX_VAR(opline->op1.var);
810 }
811 if (Z_TYPE_P(obj) != IS_OBJECT
812 || Z_OBJ_P(obj)->handlers != &std_object_handlers) {
813 break;
814 }
815 prop_info = zend_get_property_info(Z_OBJCE_P(obj), prop_name, 1);
816 if (prop_info
817 && prop_info != ZEND_WRONG_PROPERTY_INFO
818 && !(prop_info->flags & ZEND_ACC_STATIC)) {
819 val = OBJ_PROP(Z_OBJ_P(obj), prop_info->offset);
820 TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, Z_TYPE_P(val), 0, 0);
821 }
822 }
823 break;
824 default:
825 break;
826 }
827
828 if (opline->opcode == ZEND_DO_FCALL
829 || opline->opcode == ZEND_DO_ICALL
830 || opline->opcode == ZEND_DO_UCALL
831 || opline->opcode == ZEND_DO_FCALL_BY_NAME) {
832 if (ZEND_CALL_INFO(EX(call)) & ZEND_CALL_MEGAMORPHIC) {
833 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
834 break;
835 }
836 if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
837 if (EX(call)->func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) {
838 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
839 break;
840 }
841 TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func);
842 }
843 } else if (opline->opcode == ZEND_INCLUDE_OR_EVAL
844 || opline->opcode == ZEND_CALLABLE_CONVERT) {
845 /* TODO: Support tracing JIT for ZEND_CALLABLE_CONVERT. */
846 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
847 break;
848 }
849
850 handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
851 #ifdef HAVE_GCC_GLOBAL_REGS
852 handler();
853 if (UNEXPECTED(opline == zend_jit_halt_op)) {
854 stop = ZEND_JIT_TRACE_STOP_RETURN;
855 opline = NULL;
856 halt = ZEND_JIT_TRACE_HALT;
857 break;
858 }
859 if (UNEXPECTED(execute_data != prev_execute_data)) {
860 #else
861 rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);
862 if (rc != 0) {
863 if (rc < 0) {
864 stop = ZEND_JIT_TRACE_STOP_RETURN;
865 opline = NULL;
866 halt = ZEND_JIT_TRACE_HALT;
867 break;
868 } else if (execute_data == EG(current_execute_data)) {
869 /* return after interrupt handler */
870 rc = 0;
871 }
872 execute_data = EG(current_execute_data);
873 opline = EX(opline);
874 #endif
875
876 op_array = &EX(func)->op_array;
877 jit_extension =
878 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
879 if (UNEXPECTED(!jit_extension)
880 || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) {
881 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
882 break;
883 }
884 offset = jit_extension->offset;
885 if (!op_array->function_name
886 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
887 op_array = jit_extension->op_array;
888 }
889
890 #ifdef HAVE_GCC_GLOBAL_REGS
891 if (execute_data->prev_execute_data == prev_execute_data) {
892 #else
893 if (rc == 0) {
894 /* pass */
895 } else if (rc == 1) {
896 #endif
897 /* Enter into function */
898 prev_call = NULL;
899 if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) {
900 stop = ZEND_JIT_TRACE_STOP_TOO_DEEP;
901 break;
902 }
903
904 if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
905 /* TODO: Can we continue recording ??? */
906 stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
907 break;
908 }
909
910 TRACE_RECORD(ZEND_JIT_TRACE_ENTER,
911 EX(return_value) != NULL ? ZEND_JIT_TRACE_RETURN_VALUE_USED : 0,
912 op_array);
913
914 count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level);
915
916 if (opline == orig_opline) {
917 if (count + 1 >= JIT_G(max_recursive_calls)) {
918 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
919 break;
920 }
921 backtrack_recursion = idx;
922 } else if (count >= JIT_G(max_recursive_calls)) {
923 stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
924 break;
925 }
926
927 unrolled_calls[ret_level + level] = &EX(func)->op_array;
928 level++;
929 } else {
930 /* Return from function */
931 prev_call = EX(call);
932 if (level == 0) {
933 if (start == ZEND_JIT_TRACE_START_RETURN
934 && JIT_G(max_recursive_returns) > 0
935 && execute_data->prev_execute_data
936 && execute_data->prev_execute_data->func
937 && execute_data->prev_execute_data->func->type == ZEND_USER_FUNCTION
938 && zend_jit_trace_has_recursive_ret(execute_data, trace_buffer[0].op_array, orig_opline, ret_level)) {
939 if (ret_level > ZEND_JIT_TRACE_MAX_RET_DEPTH) {
940 stop = ZEND_JIT_TRACE_STOP_TOO_DEEP_RET;
941 break;
942 }
943 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
944 count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level);
945 if (opline == orig_opline) {
946 if (count + 1 >= JIT_G(max_recursive_returns)) {
947 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
948 break;
949 }
950 backtrack_ret_recursion = idx;
951 backtrack_ret_recursion_level = ret_level;
952 } else if (count >= JIT_G(max_recursive_returns)) {
953 stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
954 break;
955 }
956
957 unrolled_calls[ret_level] = &EX(func)->op_array;
958 ret_level++;
959 last_loop_opline = NULL;
960
961 if (prev_call) {
962 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0);
963 if (ret < 0) {
964 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
965 break;
966 }
967 idx = ret;
968 }
969 } else if (start & ZEND_JIT_TRACE_START_LOOP
970 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
971 ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
972 /* Fail to try close the loop.
973 If this doesn't work terminate it. */
974 stop = ZEND_JIT_TRACE_STOP_LOOP_EXIT;
975 break;
976 } else if (start & ZEND_JIT_TRACE_START_ENTER
977 && EX(prev_execute_data)
978 && EX(func) == EX(prev_execute_data)->func
979 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
980 ZEND_JIT_TRACE_STOP_RECURSION_EXIT) {
981 stop = ZEND_JIT_TRACE_STOP_RECURSION_EXIT;
982 break;
983 } else {
984 stop = ZEND_JIT_TRACE_STOP_RETURN;
985 break;
986 }
987 } else {
988 level--;
989 if (level < last_loop_level) {
990 last_loop_opline = NULL;
991 }
992 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
993 }
994 }
995 #ifdef HAVE_GCC_GLOBAL_REGS
996 prev_execute_data = execute_data;
997 #endif
998 }
999 if (EX(call) != prev_call) {
1000 if (EX(call)
1001 && EX(call)->prev_execute_data == prev_call) {
1002 zend_function *func;
1003 zend_jit_op_array_trace_extension *jit_extension;
1004
1005 if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
1006 /* TODO: Can we continue recording ??? */
1007 stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
1008 break;
1009 } else if (EX(call)->func->common.fn_flags & ZEND_ACC_NEVER_CACHE) {
1010 /* TODO: Can we continue recording ??? */
1011 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1012 break;
1013 }
1014 func = EX(call)->func;
1015 if (func->type == ZEND_INTERNAL_FUNCTION
1016 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
1017 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1018 break;
1019 }
1020 if (func->type == ZEND_USER_FUNCTION
1021 && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
1022 jit_extension =
1023 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
1024 if (UNEXPECTED(!jit_extension)
1025 || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)
1026 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) {
1027 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
1028 break;
1029 }
1030 func = (zend_function*)jit_extension->op_array;
1031 }
1032
1033 #ifndef HAVE_GCC_GLOBAL_REGS
1034 opline = EX(opline);
1035 #endif
1036
1037 if (JIT_G(max_polymorphic_calls) == 0
1038 && zend_jit_may_be_polymorphic_call(opline - 1)) {
1039 func = NULL;
1040 } else if ((is_megamorphic == ZEND_JIT_EXIT_METHOD_CALL
1041 || is_megamorphic == ZEND_JIT_EXIT_CLOSURE_CALL)
1042 && trace_buffer[1].opline == opline - 1) {
1043 func = NULL;
1044 }
1045 if (!func) {
1046 ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC);
1047 }
1048 TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func);
1049 }
1050 prev_call = EX(call);
1051 }
1052
1053 #ifndef HAVE_GCC_GLOBAL_REGS
1054 opline = EX(opline);
1055 #endif
1056
1057 if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
1058 /* Abort trace because of exception */
1059 stop = ZEND_JIT_TRACE_STOP_EXCEPTION;
1060 break;
1061 }
1062
1063 trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
1064 if (trace_flags) {
1065 if (trace_flags & ZEND_JIT_TRACE_JITED) {
1066 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
1067 if ((start & ZEND_JIT_TRACE_START_LOOP) != 0
1068 && level + ret_level == 0
1069 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1070 ZEND_JIT_TRACE_STOP_COMPILED_LOOP) {
1071 /* Fail to try close outer loop through side exit.
1072 If this doesn't work just link. */
1073 stop = ZEND_JIT_TRACE_STOP_COMPILED_LOOP;
1074 break;
1075 } else {
1076 stop = ZEND_JIT_TRACE_STOP_LINK;
1077 break;
1078 }
1079 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
1080 if (start != ZEND_JIT_TRACE_START_RETURN) {
1081 // TODO: We may try to inline function ???
1082 stop = ZEND_JIT_TRACE_STOP_LINK;
1083 break;
1084 }
1085 if (backtrack_link_to_enter < 0) {
1086 backtrack_link_to_enter = idx;
1087 link_to_enter_opline = opline;
1088 }
1089 } else {
1090 stop = ZEND_JIT_TRACE_STOP_LINK;
1091 break;
1092 }
1093 } else if (trace_flags & ZEND_JIT_TRACE_BLACKLISTED) {
1094 stop = ZEND_JIT_TRACE_STOP_BLACK_LIST;
1095 break;
1096 } else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
1097 uint8_t bad_stop;
1098
1099 if (start != ZEND_JIT_TRACE_START_SIDE) {
1100 if (opline == orig_opline && level + ret_level == 0) {
1101 stop = ZEND_JIT_TRACE_STOP_LOOP;
1102 break;
1103 }
1104 }
1105
1106 if (start != ZEND_JIT_TRACE_START_SIDE
1107 || level + ret_level != 0) {
1108 /* First try creating a trace for inner loop.
1109 If this doesn't work try loop unroling. */
1110 bad_stop = zend_jit_trace_bad_stop_event(opline,
1111 JIT_G(blacklist_root_trace) / 2);
1112 if (bad_stop != ZEND_JIT_TRACE_STOP_INNER_LOOP
1113 && bad_stop != ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
1114 if (start == ZEND_JIT_TRACE_START_SIDE
1115 || zend_jit_trace_bad_stop_event(orig_opline,
1116 JIT_G(blacklist_root_trace) / 2) != ZEND_JIT_TRACE_STOP_INNER_LOOP) {
1117 stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
1118 break;
1119 }
1120 }
1121 }
1122
1123 if (opline == last_loop_opline
1124 && level == last_loop_level) {
1125 idx = zend_jit_trace_subtrace(trace_buffer,
1126 last_loop, idx, ZEND_JIT_TRACE_START_LOOP, op_array, opline);
1127 start = ZEND_JIT_TRACE_START_LOOP;
1128 stop = ZEND_JIT_TRACE_STOP_LOOP;
1129 ret_level = 0;
1130 break;
1131 } else if (loop_unroll_limit < JIT_G(max_loop_unrolls)) {
1132 last_loop = idx;
1133 last_loop_opline = opline;
1134 last_loop_level = level;
1135 loop_unroll_limit++;
1136 } else {
1137 stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL;
1138 break;
1139 }
1140 } else if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
1141 TRACE_RECORD(ZEND_JIT_TRACE_VM, 0, opline);
1142 stop = ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
1143 break;
1144 }
1145 }
1146 }
1147
1148 end_opline = opline;
1149 if (!ZEND_JIT_TRACE_STOP_OK(stop)) {
1150 if (backtrack_recursion > 0) {
1151 idx = backtrack_recursion;
1152 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
1153 end_opline = orig_opline;
1154 } else if (backtrack_ret_recursion > 0) {
1155 idx = backtrack_ret_recursion;
1156 ret_level = backtrack_ret_recursion_level;
1157 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
1158 end_opline = orig_opline;
1159 } else if (backtrack_link_to_enter > 0) {
1160 if (stop == ZEND_JIT_TRACE_STOP_DEEP_RECURSION
1161 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) / 2) ==
1162 ZEND_JIT_TRACE_STOP_DEEP_RECURSION) {
1163 idx = backtrack_link_to_enter;
1164 stop = ZEND_JIT_TRACE_STOP_LINK;
1165 end_opline = link_to_enter_opline;
1166 }
1167 }
1168 }
1169
1170 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
1171 /* Shrink fake INIT_CALLs */
1172 while (trace_buffer[idx-1].op == ZEND_JIT_TRACE_INIT_CALL
1173 && (trace_buffer[idx-1].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1174 idx--;
1175 }
1176 }
1177
1178 TRACE_END(ZEND_JIT_TRACE_END, stop, end_opline);
1179
1180 #ifdef HAVE_GCC_GLOBAL_REGS
1181 if (!halt) {
1182 EX(opline) = opline;
1183 }
1184 #endif
1185
1186 #ifdef HAVE_GCC_GLOBAL_REGS
1187 execute_data = save_execute_data;
1188 opline = save_opline;
1189 #endif
1190
1191 return stop | halt;
1192 }
1193
