1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 | Xinchen Hui <laruence@php.net> |
17 +----------------------------------------------------------------------+
18 */
19
20 #include "Zend/zend_execute.h"
21 #include "Zend/zend_exceptions.h"
22 #include "Zend/zend_vm.h"
23 #include "Zend/zend_closures.h"
24 #include "Zend/zend_constants.h"
25 #include "Zend/zend_API.h"
26
27 #include <ZendAccelerator.h>
28 #include "Optimizer/zend_func_info.h"
29 #include "Optimizer/zend_call_graph.h"
30 #include "zend_jit.h"
31
32 #include "zend_jit_internal.h"
33
34 #ifdef HAVE_GCC_GLOBAL_REGS
35 # pragma GCC diagnostic ignored "-Wvolatile-register-var"
36 # if defined(__x86_64__)
37 register zend_execute_data* volatile execute_data __asm__("%r14");
38 register const zend_op* volatile opline __asm__("%r15");
39 # elif defined(i386)
40 register zend_execute_data* volatile execute_data __asm__("%esi");
41 register const zend_op* volatile opline __asm__("%edi");
42 # elif defined(__aarch64__)
43 register zend_execute_data* volatile execute_data __asm__("x27");
44 register const zend_op* volatile opline __asm__("x28");
45 # endif
46 # pragma GCC diagnostic warning "-Wvolatile-register-var"
47 #endif
48
zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)49 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)
50 {
51 zend_execute_data *old_execute_data;
52
53 if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
54 zend_clean_and_cache_symbol_table(EX(symbol_table));
55 }
56
57 zend_vm_stack_free_extra_args_ex(call_info, execute_data);
58 if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) {
59 OBJ_RELEASE(Z_OBJ(execute_data->This));
60 } else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
61 OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
62 }
63 if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
64 zend_free_extra_named_params(EX(extra_named_params));
65 }
66
67 old_execute_data = execute_data;
68 execute_data = EX(prev_execute_data);
69 zend_vm_stack_free_call_frame_ex(call_info, old_execute_data);
70
71 if (UNEXPECTED(EG(exception) != NULL)) {
72 const zend_op *old_opline = EX(opline);
73 zend_throw_exception_internal(NULL);
74 if (old_opline->result_type != IS_UNDEF) {
75 zval_ptr_dtor(EX_VAR(old_opline->result.var));
76 }
77 #ifndef HAVE_GCC_GLOBAL_REGS
78 return 2; // ZEND_VM_LEAVE
79 #endif
80 } else {
81 EX(opline)++;
82 #ifdef HAVE_GCC_GLOBAL_REGS
83 opline = EX(opline);
84 #else
85 return 2; // ZEND_VM_LEAVE
86 #endif
87 }
88 }
89
zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)90 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)
91 {
92 if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) {
93 if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
94 zend_clean_and_cache_symbol_table(EX(symbol_table));
95 }
96 zend_vm_stack_free_extra_args_ex(call_info, execute_data);
97 }
98 if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
99 zend_free_extra_named_params(EX(extra_named_params));
100 }
101 if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
102 OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
103 }
104 execute_data = EG(current_execute_data);
105 #ifdef HAVE_GCC_GLOBAL_REGS
106 opline = zend_jit_halt_op;
107 #else
108 return -1; // ZEND_VM_RETURN
109 #endif
110 }
111
zend_jit_leave_func_helper(EXECUTE_DATA_D)112 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(EXECUTE_DATA_D)
113 {
114 uint32_t call_info = EX_CALL_INFO();
115
116 if (call_info & ZEND_CALL_TOP) {
117 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info);
118 } else {
119 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info);
120 }
121 }
122
zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)123 void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)
124 {
125 zend_op_array *op_array = &EX(func)->op_array;
126
127 if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) {
128 uint32_t first_extra_arg = op_array->num_args;
129 uint32_t num_args = EX_NUM_ARGS();
130 zval *end, *src, *dst;
131 uint32_t type_flags = 0;
132
133 if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) {
134 /* Skip useless ZEND_RECV and ZEND_RECV_INIT opcodes */
135 #ifdef HAVE_GCC_GLOBAL_REGS
136 opline += first_extra_arg;
137 #else
138 EX(opline) += first_extra_arg;
139 #endif
140 }
141
142 /* move extra args into separate array after all CV and TMP vars */
143 end = EX_VAR_NUM(first_extra_arg - 1);
144 src = end + (num_args - first_extra_arg);
145 dst = src + (op_array->last_var + op_array->T - first_extra_arg);
146 if (EXPECTED(src != dst)) {
147 do {
148 type_flags |= Z_TYPE_INFO_P(src);
149 ZVAL_COPY_VALUE(dst, src);
150 ZVAL_UNDEF(src);
151 src--;
152 dst--;
153 } while (src != end);
154 if (type_flags & (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT)) {
155 ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
156 }
157 } else {
158 do {
159 if (Z_REFCOUNTED_P(src)) {
160 ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
161 break;
162 }
163 src--;
164 } while (src != end);
165 }
166 }
167 }
168
zend_jit_deprecated_helper(OPLINE_D)169 bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D)
170 {
171 zend_execute_data *call = (zend_execute_data *) opline;
172 zend_function *fbc = call->func;
173
174 zend_deprecated_function(fbc);
175
176 if (EG(exception)) {
177 #ifndef HAVE_GCC_GLOBAL_REGS
178 zend_execute_data *execute_data = EG(current_execute_data);
179 #endif
180 const zend_op *opline = EG(opline_before_exception);
181 if (opline && RETURN_VALUE_USED(opline)) {
182 ZVAL_UNDEF(EX_VAR(opline->result.var));
183 }
184
185 zend_vm_stack_free_args(call);
186
187 if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) {
188 OBJ_RELEASE(Z_OBJ(call->This));
189 }
190
191 zend_vm_stack_free_call_frame(call);
192 return 0;
193 }
194 return 1;
195 }
196
zend_jit_undefined_long_key(EXECUTE_DATA_D)197 void ZEND_FASTCALL zend_jit_undefined_long_key(EXECUTE_DATA_D)
198 {
199 const zend_op *opline = EX(opline);
200 zval *result = EX_VAR(opline->result.var);
201 zval *dim;
202
203 if (opline->op2_type == IS_CONST) {
204 dim = RT_CONSTANT(opline, opline->op2);
205 } else {
206 dim = EX_VAR(opline->op2.var);
207 }
208 ZEND_ASSERT(Z_TYPE_P(dim) == IS_LONG);
209 zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, Z_LVAL_P(dim));
210 ZVAL_NULL(result);
211 }
212
zend_jit_undefined_long_key_ex(zend_long key EXECUTE_DATA_DC)213 void ZEND_FASTCALL zend_jit_undefined_long_key_ex(zend_long key EXECUTE_DATA_DC)
214 {
215 const zend_op *opline = EX(opline);
216 zval *result = EX_VAR(opline->result.var);
217
218 zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, key);
219 ZVAL_NULL(result);
220 }
221
zend_jit_undefined_string_key(EXECUTE_DATA_D)222 void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D)
223 {
224 const zend_op *opline = EX(opline);
225 zval *result = EX_VAR(opline->result.var);
226 zval *dim;
227 zend_ulong lval;
228
229 if (opline->op2_type == IS_CONST) {
230 dim = RT_CONSTANT(opline, opline->op2);
231 } else {
232 dim = EX_VAR(opline->op2.var);
233 }
234 ZEND_ASSERT(Z_TYPE_P(dim) == IS_STRING);
235 if (ZEND_HANDLE_NUMERIC(Z_STR_P(dim), lval)) {
236 zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, lval);
237 } else {
238 zend_error(E_WARNING, "Undefined array key \"%s\"", Z_STRVAL_P(dim));
239 }
240 ZVAL_NULL(result);
241 }
242
zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)243 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)
244 {
245 zend_op_array *op_array = (zend_op_array*)EX(func);
246 zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
247 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t) jit_extension->orig_handler;
248 ++*(uintptr_t*)(EX(run_time_cache) + zend_jit_profile_counter_rid);
249 ++zend_jit_profile_counter;
250 ZEND_OPCODE_TAIL_CALL(handler);
251 }
252
zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)253 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
254 {
255 zend_jit_op_array_hot_extension *jit_extension =
256 (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
257 #ifndef HAVE_GCC_GLOBAL_REGS
258 const zend_op *opline = EX(opline);
259 #endif
260
261 *(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func));
262
263 if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
264 *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
265 zend_jit_hot_func(execute_data, opline);
266 ZEND_OPCODE_RETURN();
267 } else {
268 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
269 ZEND_OPCODE_TAIL_CALL(handler);
270 }
271 }
272
zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)273 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
274 {
275 zend_jit_op_array_hot_extension *jit_extension =
276 (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
277 #ifndef HAVE_GCC_GLOBAL_REGS
278 const zend_op *opline = EX(opline);
279 #endif
280
281 *(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop));
282
283 if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
284 *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
285 zend_jit_hot_func(execute_data, opline);
286 ZEND_OPCODE_RETURN();
287 } else {
288 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
289 ZEND_OPCODE_TAIL_CALL(handler);
290 }
291 }
292
_zend_quick_get_constant(const zval * key,uint32_t flags,int check_defined_only)293 static zend_always_inline zend_constant* _zend_quick_get_constant(
294 const zval *key, uint32_t flags, int check_defined_only)
295 {
296 #ifndef HAVE_GCC_GLOBAL_REGS
297 zend_execute_data *execute_data = EG(current_execute_data);
298 #endif
299 const zend_op *opline = EX(opline);
300 zval *zv;
301 zend_constant *c = NULL;
302
303 /* null/true/false are resolved during compilation, so don't check for them here. */
304 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
305 if (zv) {
306 c = (zend_constant*)Z_PTR_P(zv);
307 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
308 key++;
309 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
310 if (zv) {
311 c = (zend_constant*)Z_PTR_P(zv);
312 }
313 }
314
315 if (!c) {
316 if (!check_defined_only) {
317 zend_throw_error(NULL, "Undefined constant \"%s\"", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)));
318 ZVAL_UNDEF(EX_VAR(opline->result.var));
319 }
320 CACHE_PTR(opline->extended_value, ENCODE_SPECIAL_CACHE_NUM(zend_hash_num_elements(EG(zend_constants))));
321 return NULL;
322 }
323
324 if (!check_defined_only) {
325 if (ZEND_CONSTANT_FLAGS(c) & CONST_DEPRECATED) {
326 zend_error(E_DEPRECATED, "Constant %s is deprecated", ZSTR_VAL(c->name));
327 if (EG(exception)) {
328 return NULL;
329 }
330 return c;
331 }
332 }
333
334 CACHE_PTR(opline->extended_value, c);
335 return c;
336 }
337
zend_jit_get_constant(const zval * key,uint32_t flags)338 zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags)
339 {
340 return _zend_quick_get_constant(key, flags, 0);
341 }
342
zend_jit_check_constant(const zval * key)343 zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key)
344 {
345 return _zend_quick_get_constant(key, 0, 1);
346 }
347
zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)348 static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)
349 {
350 zend_jit_op_array_trace_extension *jit_extension =
351 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
352 size_t offset = jit_extension->offset;
353 #ifndef HAVE_GCC_GLOBAL_REGS
354 const zend_op *opline = EX(opline);
355 #endif
356
357 *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
358
359 if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
360 *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT;
361 if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
362 #ifdef HAVE_GCC_GLOBAL_REGS
363 opline = NULL;
364 return;
365 #else
366 return -1;
367 #endif
368 }
369 #ifdef HAVE_GCC_GLOBAL_REGS
370 execute_data = EG(current_execute_data);
371 opline = execute_data ? EX(opline) : NULL;
372 return;
373 #else
374 return 1;
375 #endif
376 } else {
377 zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
378 ZEND_OPCODE_TAIL_CALL(handler);
379 }
380 }
381
zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)382 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
383 {
384 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
385 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
386 }
387
zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)388 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
389 {
390 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
391 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
392 }
393
zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)394 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
395 {
396 ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
397 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
398 }
399
400 #define TRACE_RECORD(_op, _info, _ptr) \
401 trace_buffer[idx].info = _op | (_info); \
402 trace_buffer[idx].ptr = _ptr; \
403 idx++; \
404 if (idx >= JIT_G(max_trace_length) - 2) { \
405 stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
406 break; \
407 }
408
409 #define TRACE_RECORD_VM(_op, _ptr, _op1_type, _op2_type, _op3_type) \
410 trace_buffer[idx].op = _op; \
411 trace_buffer[idx].op1_type = _op1_type; \
412 trace_buffer[idx].op2_type = _op2_type; \
413 trace_buffer[idx].op3_type = _op3_type; \
414 trace_buffer[idx].ptr = _ptr; \
415 idx++; \
416 if (idx >= JIT_G(max_trace_length) - 2) { \
417 stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
418 break; \
419 }
420
421 #define TRACE_START(_op, _start, _ptr1, _ptr2) \
422 trace_buffer[0].op = _op; \
423 trace_buffer[0].start = _start; \
424 trace_buffer[0].level = 0; \
425 trace_buffer[0].ptr = _ptr1; \
426 trace_buffer[1].last = 0; \
427 trace_buffer[1].ptr = _ptr2; \
428 idx = ZEND_JIT_TRACE_START_REC_SIZE;
429
430 #define TRACE_END(_op, _stop, _ptr) \
431 trace_buffer[1].last = idx; \
432 trace_buffer[idx].op = _op; \
433 trace_buffer[idx].start = trace_buffer[idx].start; \
434 trace_buffer[idx].stop = trace_buffer[0].stop = _stop; \
435 trace_buffer[idx].level = trace_buffer[0].level = ret_level ? ret_level + 1 : 0; \
436 trace_buffer[idx].ptr = _ptr;
437
zend_jit_trace_recursive_call_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level,int level)438 static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level)
439 {
440 int i;
441 int count = 0;
442
443 for (i = ret_level; i < level; i++) {
444 count += (unrolled_calls[i] == op_array);
445 }
446 return count;
447 }
448
zend_jit_trace_recursive_ret_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level)449 static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level)
450 {
451 int i;
452 int count = 0;
453
454 for (i = 0; i < ret_level; i++) {
455 count += (unrolled_calls[i] == op_array);
456 }
457 return count;
458 }
459
zend_jit_trace_has_recursive_ret(zend_execute_data * ex,const zend_op_array * orig_op_array,const zend_op * orig_opline,int ret_level)460 static int zend_jit_trace_has_recursive_ret(zend_execute_data *ex, const zend_op_array *orig_op_array, const zend_op *orig_opline, int ret_level)
461 {
462 while (ex != NULL && ex->func != NULL && ret_level < ZEND_JIT_TRACE_MAX_RET_DEPTH) {
463 if (&ex->func->op_array == orig_op_array && ex->opline + 1 == orig_opline) {
464 return 1;
465 }
466 ex = ex->prev_execute_data;
467 ret_level++;
468 }
469 return 0;
470 }
471
zend_jit_trace_bad_stop_event(const zend_op * opline,int count)472 static uint8_t zend_jit_trace_bad_stop_event(const zend_op *opline, int count)
473 {
474 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
475 uint8_t *cache_count = JIT_G(bad_root_cache_count);
476 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
477 uint32_t i;
478
479 if (count < 0) {
480 count = 0;
481 }
482 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
483 if (cache_opline[i] == opline) {
484 if (cache_count[i] >= count) {
485 return cache_stop[i];
486 }
487 break;
488 }
489 }
490 return 0;
491 }
492
493 #define ZEND_CALL_MEGAMORPHIC ZEND_CALL_JIT_RESERVED
494
zend_jit_trace_record_fake_init_call_ex(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic,uint32_t init_level)495 static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t init_level)
496 {
497 zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
498
499 do {
500 zend_function *func;
501 zend_jit_op_array_trace_extension *jit_extension;
502
503 if (call->prev_execute_data) {
504 idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, init_level + 1);
505 if (idx < 0) {
506 return idx;
507 }
508 }
509
510 func = call->func;
511 if (func->type == ZEND_INTERNAL_FUNCTION
512 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
513 func = NULL;
514 } else if (func->type == ZEND_USER_FUNCTION) {
515 jit_extension =
516 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
517 if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE))
518 || (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))
519 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) {
520 func = NULL;
521 } else if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) {
522 func = (zend_function*)jit_extension->op_array;
523 }
524 }
525
526 if (!func
527 || (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)
528 || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE)
529 || func->common.prop_info) {
530 /* continue recording */
531 func = NULL;
532 } else if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM
533 /* TODO: use more accurate check ??? */
534 && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
535 || func->common.scope)) {
536 func = NULL;
537 ZEND_ADD_CALL_FLAG(call, ZEND_CALL_MEGAMORPHIC);
538 }
539 TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INFO(init_level), func);
540 } while (0);
541 return idx;
542 }
543
zend_jit_trace_record_fake_init_call(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic)544 static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic)
545 {
546 return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, 0);
547 }
548
zend_jit_trace_subtrace(zend_jit_trace_rec * trace_buffer,int start,int end,uint8_t event,const zend_op_array * op_array,const zend_op * opline)549 static int zend_jit_trace_subtrace(zend_jit_trace_rec *trace_buffer, int start, int end, uint8_t event, const zend_op_array *op_array, const zend_op *opline)
550 {
551 int idx;
552
553 TRACE_START(ZEND_JIT_TRACE_START, event, op_array, opline);
554 memmove(trace_buffer + idx, trace_buffer + start, (end - start) * sizeof(zend_jit_trace_rec));
555 return idx + (end - start);
556 }
557
558 /*
559 * Trace Linking Rules
560 * ===================
561 *
562 * flags
563 * +----------+----------+----------++----------+----------+----------+
564 * | || JIT |
565 * +----------+----------+----------++----------+----------+----------+
566 * start | LOOP | ENTER | RETURN || LOOP | ENTER | RETURN |
567 * +========+==========+==========+==========++==========+==========+==========+
568 * | LOOP | loop | | loop-ret || COMPILED | LINK | LINK |
569 * +--------+----------+----------+----------++----------+----------+----------+
570 * | ENTER |INNER_LOOP| rec-call | return || LINK | LINK | LINK |
571 * +--------+----------+----------+----------++----------+----------+----------+
572 * | RETURN |INNER_LOOP| | rec-ret || LINK | | LINK |
573 * +--------+----------+----------+----------++----------+----------+----------+
574 * | SIDE | unroll | | side-ret || LINK | LINK | LINK |
575 * +--------+----------+----------+----------++----------+----------+----------+
576 *
577 * loop: LOOP if "cycle" and level == 0, otherwise INNER_LOOP
578 * INNER_LOOP: abort recording and start new one (wait for loop)
579 * COMPILED: abort recording (wait while side exit creates outer loop)
580 * unroll: continue recording while unroll limit reached
581 * rec-call: RECURSIVE_CALL if "cycle" and level > N, otherwise continue
582 * loop-ret: LOOP_EXIT if level == 0, otherwise continue (wait for loop)
583 * return: RETURN if level == 0
584 * rec_ret: RECURSIVE_RET if "cycle" and ret_level > N, otherwise continue
585 * side_ret: RETURN if level == 0 && ret_level == ret_depth, otherwise continue
586 *
587 */
588
zend_jit_trace_execute(zend_execute_data * ex,const zend_op * op,zend_jit_trace_rec * trace_buffer,uint8_t start,uint32_t is_megamorphic,int ret_depth)589 zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
590 const zend_op *op,
591 zend_jit_trace_rec *trace_buffer,
592 uint8_t start,
593 uint32_t is_megamorphic,
594 int ret_depth)
595
596 {
597 #ifdef HAVE_GCC_GLOBAL_REGS
598 zend_execute_data *save_execute_data = execute_data;
599 const zend_op *save_opline = opline;
600 #endif
601 const zend_op *orig_opline, *end_opline;
602 zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
603 zend_jit_trace_stop halt = 0;
604 int level = 0;
605 int ret_level = 0;
606 zend_vm_opcode_handler_t handler;
607 const zend_op_array *op_array;
608 zend_jit_op_array_trace_extension *jit_extension;
609 size_t offset;
610 int idx, count;
611 uint8_t trace_flags, op1_type, op2_type, op3_type;
612 zend_class_entry *ce1, *ce2;
613 const zend_op *link_to_enter_opline = NULL;
614 int backtrack_link_to_enter = -1;
615 int backtrack_recursion = -1;
616 int backtrack_ret_recursion = -1;
617 int backtrack_ret_recursion_level = 0;
618 int loop_unroll_limit = 0;
619 int last_loop = -1;
620 int last_loop_level = -1;
621 const zend_op *last_loop_opline = NULL;
622 const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
623 #ifdef HAVE_GCC_GLOBAL_REGS
624 zend_execute_data *prev_execute_data = ex;
625
626 execute_data = ex;
627 opline = EX(opline) = op;
628 #else
629 int rc;
630 zend_execute_data *execute_data = ex;
631 const zend_op *opline = EX(opline);
632 #endif
633 zend_execute_data *prev_call = EX(call);
634
635 orig_opline = opline;
636
637 op_array = &EX(func)->op_array;
638 jit_extension =
639 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
640 offset = jit_extension->offset;
641 if (!op_array->function_name
642 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
643 op_array = jit_extension->op_array;
644 }
645
646 TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline);
647
648 if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
649 /* Abort trace because of exception */
650 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_EXCEPTION, opline);
651 #ifdef HAVE_GCC_GLOBAL_REGS
652 execute_data = save_execute_data;
653 opline = save_opline;
654 #endif
655 return ZEND_JIT_TRACE_STOP_EXCEPTION;
656 }
657
658 trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
659 if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
660 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_NOT_SUPPORTED, opline);
661 #ifdef HAVE_GCC_GLOBAL_REGS
662 execute_data = save_execute_data;
663 opline = save_opline;
664 #endif
665 return ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
666 }
667
668 if (prev_call) {
669 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic);
670 if (ret < 0) {
671 TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_BAD_FUNC, opline);
672 #ifdef HAVE_GCC_GLOBAL_REGS
673 execute_data = save_execute_data;
674 opline = save_opline;
675 #endif
676 return ZEND_JIT_TRACE_STOP_BAD_FUNC;
677 }
678 idx = ret;
679 }
680
681 while (1) {
682 ce1 = ce2 = NULL;
683 op1_type = op2_type = op3_type = IS_UNKNOWN;
684 if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV))
685 && opline->opcode != ZEND_ROPE_ADD
686 && opline->opcode != ZEND_ROPE_END
687 && opline->opcode != ZEND_NEW
688 && opline->opcode != ZEND_FETCH_CLASS_CONSTANT
689 && opline->opcode != ZEND_INIT_STATIC_METHOD_CALL) {
690 zval *zv = EX_VAR(opline->op1.var);
691 op1_type = Z_TYPE_P(zv);
692 uint8_t flags = 0;
693
694 if (op1_type == IS_INDIRECT) {
695 zv = Z_INDIRECT_P(zv);
696 op1_type = Z_TYPE_P(zv);
697 flags |= IS_TRACE_INDIRECT;
698 }
699 if (op1_type == IS_REFERENCE) {
700 zv = Z_REFVAL_P(zv);
701 op1_type = Z_TYPE_P(zv);
702 flags |= IS_TRACE_REFERENCE;
703 }
704 if (Z_TYPE_P(zv) == IS_OBJECT) {
705 ce1 = Z_OBJCE_P(zv);
706 } else if (Z_TYPE_P(zv) == IS_ARRAY) {
707 if (HT_IS_PACKED(Z_ARRVAL_P(zv))) {
708 flags |= IS_TRACE_PACKED;
709 }
710 }
711 op1_type |= flags;
712 } else if (opline->op1_type == IS_UNUSED && (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
713 uint32_t op1_flags = ZEND_VM_OP1_FLAGS(zend_get_opcode_flags(opline->opcode));
714 if ((op1_flags & ZEND_VM_OP_MASK) == ZEND_VM_OP_THIS) {
715 op1_type = IS_OBJECT;
716 ce1 = Z_OBJCE(EX(This));
717 }
718 }
719 if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)
720 && opline->opcode != ZEND_INSTANCEOF
721 && opline->opcode != ZEND_UNSET_STATIC_PROP
722 && opline->opcode != ZEND_ISSET_ISEMPTY_STATIC_PROP
723 && opline->opcode != ZEND_ASSIGN_STATIC_PROP
724 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_REF
725 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_OP
726 && opline->opcode != ZEND_PRE_INC_STATIC_PROP
727 && opline->opcode != ZEND_POST_INC_STATIC_PROP
728 && opline->opcode != ZEND_PRE_DEC_STATIC_PROP
729 && opline->opcode != ZEND_POST_DEC_STATIC_PROP
730 && opline->opcode != ZEND_FETCH_STATIC_PROP_R
731 && opline->opcode != ZEND_FETCH_STATIC_PROP_W
732 && opline->opcode != ZEND_FETCH_STATIC_PROP_RW
733 && opline->opcode != ZEND_FETCH_STATIC_PROP_IS
734 && opline->opcode != ZEND_FETCH_STATIC_PROP_FUNC_ARG
735 && opline->opcode != ZEND_FETCH_STATIC_PROP_UNSET
736 && (opline->op2_type == IS_CV
737 || (opline->opcode != ZEND_FE_FETCH_R
738 && opline->opcode != ZEND_FE_FETCH_RW))) {
739 zval *zv = EX_VAR(opline->op2.var);
740 uint8_t flags = 0;
741
742 op2_type = Z_TYPE_P(zv);
743 if (op2_type == IS_INDIRECT) {
744 zv = Z_INDIRECT_P(zv);
745 op2_type = Z_TYPE_P(zv);
746 flags |= IS_TRACE_INDIRECT;
747 }
748 if (op2_type == IS_REFERENCE) {
749 zv = Z_REFVAL_P(zv);
750 op2_type = Z_TYPE_P(zv);
751 flags |= IS_TRACE_REFERENCE;
752 }
753 if (Z_TYPE_P(zv) == IS_OBJECT) {
754 ce2 = Z_OBJCE_P(zv);
755 }
756 op2_type |= flags;
757 }
758 if (opline->opcode == ZEND_ASSIGN_DIM ||
759 opline->opcode == ZEND_ASSIGN_OBJ ||
760 opline->opcode == ZEND_ASSIGN_STATIC_PROP ||
761 opline->opcode == ZEND_ASSIGN_DIM_OP ||
762 opline->opcode == ZEND_ASSIGN_OBJ_OP ||
763 opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP ||
764 opline->opcode == ZEND_ASSIGN_OBJ_REF ||
765 opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF) {
766 if ((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
767 zval *zv = EX_VAR((opline+1)->op1.var);
768 uint8_t flags = 0;
769
770 op3_type = Z_TYPE_P(zv);
771 if (op3_type == IS_INDIRECT) {
772 zv = Z_INDIRECT_P(zv);
773 op3_type = Z_TYPE_P(zv);
774 flags |= IS_TRACE_INDIRECT;
775 }
776 if (op3_type == IS_REFERENCE) {
777 zv = Z_REFVAL_P(zv);
778 op3_type = Z_TYPE_P(zv);
779 flags |= IS_TRACE_REFERENCE;
780 }
781 op3_type |= flags;
782 }
783 }
784
785 TRACE_RECORD_VM(ZEND_JIT_TRACE_VM, opline, op1_type, op2_type, op3_type);
786
787 if (ce1) {
788 TRACE_RECORD(ZEND_JIT_TRACE_OP1_TYPE, 0, ce1);
789 }
790
791 if (ce2) {
792 TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2);
793 }
794
795 switch (opline->opcode) {
796 case ZEND_FETCH_DIM_R:
797 case ZEND_FETCH_DIM_W:
798 case ZEND_FETCH_DIM_RW:
799 case ZEND_FETCH_DIM_IS:
800 case ZEND_FETCH_DIM_FUNC_ARG:
801 case ZEND_FETCH_DIM_UNSET:
802 case ZEND_FETCH_LIST_R:
803 case ZEND_FETCH_LIST_W:
804 case ZEND_ASSIGN_DIM:
805 case ZEND_ASSIGN_DIM_OP:
806 case ZEND_UNSET_DIM:
807 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
808 if (opline->op1_type == IS_CONST) {
809 zval *arr = RT_CONSTANT(opline, opline->op1);
810 op1_type = Z_TYPE_P(arr);
811 }
812 if ((op1_type & IS_TRACE_TYPE_MASK) == IS_ARRAY
813 && opline->op2_type != IS_UNDEF) {
814 zval *arr, *dim, *val;
815 uint8_t val_type = IS_UNDEF;
816
817 if (opline->op2_type == IS_CONST) {
818 dim = RT_CONSTANT(opline, opline->op2);
819 } else {
820 dim = EX_VAR(opline->op2.var);
821 }
822
823 if (Z_TYPE_P(dim) == IS_LONG || Z_TYPE_P(dim) == IS_STRING) {
824 if (opline->op1_type == IS_CONST) {
825 arr = RT_CONSTANT(opline, opline->op1);
826 } else {
827 arr = EX_VAR(opline->op1.var);
828 }
829 if (Z_TYPE_P(arr) == IS_INDIRECT) {
830 arr = Z_INDIRECT_P(arr);
831 }
832 if (Z_TYPE_P(arr) == IS_REFERENCE) {
833 arr = Z_REFVAL_P(arr);
834 }
835 ZEND_ASSERT(Z_TYPE_P(arr) == IS_ARRAY);
836 if (Z_TYPE_P(dim) == IS_LONG) {
837 val = zend_hash_index_find(Z_ARRVAL_P(arr), Z_LVAL_P(dim));
838 } else /*if Z_TYPE_P(dim) == IS_STRING)*/ {
839 val = zend_symtable_find(Z_ARRVAL_P(arr), Z_STR_P(dim));
840 }
841 if (val) {
842 val_type = Z_TYPE_P(val);
843 }
844 TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, val_type, 0, 0);
845 }
846 }
847 break;
848 case ZEND_FETCH_OBJ_R: {
849 if (opline->op2_type == IS_CONST) {
850 /* Remove the SIMPLE_GET flag to avoid inlining hooks. */
851 void **cache_slot = CACHE_ADDR(opline->extended_value & ~ZEND_FETCH_REF);
852 uintptr_t prop_offset = (uintptr_t)CACHED_PTR_EX(cache_slot + 1);
853 if (IS_HOOKED_PROPERTY_OFFSET(prop_offset)) {
854 CACHE_PTR_EX(cache_slot + 1, (void*)((uintptr_t)CACHED_PTR_EX(cache_slot + 1) & ~ZEND_PROPERTY_HOOK_SIMPLE_GET_BIT)); \
855 }
856 }
857 ZEND_FALLTHROUGH;
858 }
859 case ZEND_FETCH_OBJ_W:
860 case ZEND_FETCH_OBJ_RW:
861 case ZEND_FETCH_OBJ_IS:
862 case ZEND_FETCH_OBJ_FUNC_ARG:
863 case ZEND_FETCH_OBJ_UNSET:
864 case ZEND_ASSIGN_OBJ:
865 case ZEND_ASSIGN_OBJ_OP:
866 case ZEND_ASSIGN_OBJ_REF:
867 case ZEND_UNSET_OBJ:
868 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
869 case ZEND_PRE_INC_OBJ:
870 case ZEND_PRE_DEC_OBJ:
871 case ZEND_POST_INC_OBJ:
872 case ZEND_POST_DEC_OBJ:
873 if (opline->op1_type != IS_CONST
874 && opline->op2_type == IS_CONST
875 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
876 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
877 zval *obj, *val;
878 zend_string *prop_name = Z_STR_P(RT_CONSTANT(opline, opline->op2));
879 zend_property_info *prop_info;
880
881 if (opline->op1_type == IS_UNUSED) {
882 obj = &EX(This);
883 } else {
884 obj = EX_VAR(opline->op1.var);
885 }
886 if (Z_TYPE_P(obj) != IS_OBJECT
887 || Z_OBJ_P(obj)->handlers != &std_object_handlers) {
888 break;
889 }
890 prop_info = zend_get_property_info(Z_OBJCE_P(obj), prop_name, 1);
891 if (prop_info
892 && prop_info != ZEND_WRONG_PROPERTY_INFO
893 && !prop_info->hooks
894 && !(prop_info->flags & ZEND_ACC_STATIC)) {
895 val = OBJ_PROP(Z_OBJ_P(obj), prop_info->offset);
896 TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, Z_TYPE_P(val), 0, 0);
897 }
898 }
899 break;
900 default:
901 break;
902 }
903
904 if (opline->opcode == ZEND_DO_FCALL
905 || opline->opcode == ZEND_DO_ICALL
906 || opline->opcode == ZEND_DO_UCALL
907 || opline->opcode == ZEND_DO_FCALL_BY_NAME) {
908 if (ZEND_CALL_INFO(EX(call)) & ZEND_CALL_MEGAMORPHIC) {
909 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
910 break;
911 }
912 if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
913 zend_function *func = EX(call)->func;
914
915 if ((func->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)
916 || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE)
917 || func->common.prop_info) {
918 /* continue recording */
919 func = NULL;
920 } else if (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) {
921 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
922 break;
923 }
924 TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, func);
925 }
926 } else if (opline->opcode == ZEND_INCLUDE_OR_EVAL
927 || opline->opcode == ZEND_CALLABLE_CONVERT) {
928 /* TODO: Support tracing JIT for ZEND_CALLABLE_CONVERT. */
929 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
930 break;
931 }
932
933 handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
934 #ifdef HAVE_GCC_GLOBAL_REGS
935 handler();
936 if (UNEXPECTED(opline == zend_jit_halt_op)) {
937 stop = ZEND_JIT_TRACE_STOP_RETURN;
938 opline = NULL;
939 halt = ZEND_JIT_TRACE_HALT;
940 break;
941 }
942 if (UNEXPECTED(execute_data != prev_execute_data)) {
943 #else
944 rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);
945 if (rc != 0) {
946 if (rc < 0) {
947 stop = ZEND_JIT_TRACE_STOP_RETURN;
948 opline = NULL;
949 halt = ZEND_JIT_TRACE_HALT;
950 break;
951 } else if (execute_data == EG(current_execute_data)) {
952 /* return after interrupt handler */
953 rc = 0;
954 }
955 execute_data = EG(current_execute_data);
956 opline = EX(opline);
957 #endif
958
959 op_array = &EX(func)->op_array;
960 jit_extension =
961 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
962 if (UNEXPECTED(!jit_extension)
963 || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) {
964 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
965 break;
966 }
967 offset = jit_extension->offset;
968 if (!op_array->function_name
969 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
970 op_array = jit_extension->op_array;
971 }
972
973 #ifdef HAVE_GCC_GLOBAL_REGS
974 if (execute_data->prev_execute_data == prev_execute_data) {
975 #else
976 if (rc == 0) {
977 /* pass */
978 } else if (rc == 1) {
979 #endif
980 /* Enter into function */
981 prev_call = NULL;
982 if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) {
983 stop = ZEND_JIT_TRACE_STOP_TOO_DEEP;
984 break;
985 }
986
987 if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
988 stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
989 break;
990 }
991
992 if (EX(func)->op_array.prop_info) {
993 stop = ZEND_JIT_TRACE_STOP_PROP_HOOK_CALL;
994 break;
995 }
996
997 if (EX(func)->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE) {
998 stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
999 break;
1000 }
1001
1002 TRACE_RECORD(ZEND_JIT_TRACE_ENTER,
1003 EX(return_value) != NULL ? ZEND_JIT_TRACE_RETURN_VALUE_USED : 0,
1004 op_array);
1005
1006 count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level);
1007
1008 if (opline == orig_opline) {
1009 if (count + 1 >= JIT_G(max_recursive_calls)) {
1010 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
1011 break;
1012 }
1013 backtrack_recursion = idx;
1014 } else if (count >= JIT_G(max_recursive_calls)) {
1015 stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
1016 break;
1017 }
1018
1019 unrolled_calls[ret_level + level] = &EX(func)->op_array;
1020 level++;
1021 } else {
1022 /* Return from function */
1023 prev_call = EX(call);
1024 if (level == 0) {
1025 if (start == ZEND_JIT_TRACE_START_RETURN
1026 && JIT_G(max_recursive_returns) > 0
1027 && execute_data->prev_execute_data
1028 && execute_data->prev_execute_data->func
1029 && execute_data->prev_execute_data->func->type == ZEND_USER_FUNCTION
1030 && zend_jit_trace_has_recursive_ret(execute_data, trace_buffer[0].op_array, orig_opline, ret_level)) {
1031 if (ret_level > ZEND_JIT_TRACE_MAX_RET_DEPTH) {
1032 stop = ZEND_JIT_TRACE_STOP_TOO_DEEP_RET;
1033 break;
1034 }
1035 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
1036 count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level);
1037 if (opline == orig_opline) {
1038 if (count + 1 >= JIT_G(max_recursive_returns)) {
1039 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
1040 break;
1041 }
1042 backtrack_ret_recursion = idx;
1043 backtrack_ret_recursion_level = ret_level;
1044 } else if (count >= JIT_G(max_recursive_returns)) {
1045 stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
1046 break;
1047 }
1048
1049 unrolled_calls[ret_level] = &EX(func)->op_array;
1050 ret_level++;
1051 last_loop_opline = NULL;
1052
1053 if (prev_call) {
1054 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0);
1055 if (ret < 0) {
1056 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1057 break;
1058 }
1059 idx = ret;
1060 }
1061 } else if (start & ZEND_JIT_TRACE_START_LOOP
1062 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1063 ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
1064 /* Fail to try close the loop.
1065 If this doesn't work terminate it. */
1066 stop = ZEND_JIT_TRACE_STOP_LOOP_EXIT;
1067 break;
1068 } else if (start & ZEND_JIT_TRACE_START_ENTER
1069 && EX(prev_execute_data)
1070 && EX(func) == EX(prev_execute_data)->func
1071 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1072 ZEND_JIT_TRACE_STOP_RECURSION_EXIT) {
1073 stop = ZEND_JIT_TRACE_STOP_RECURSION_EXIT;
1074 break;
1075 } else if ((start & ZEND_JIT_TRACE_START_SIDE)
1076 && ret_level < ret_depth) {
1077 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
1078 ret_level++;
1079 last_loop_opline = NULL;
1080
1081 if (prev_call) {
1082 int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0);
1083 if (ret < 0) {
1084 stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1085 break;
1086 }
1087 idx = ret;
1088 }
1089 } else {
1090 stop = ZEND_JIT_TRACE_STOP_RETURN;
1091 break;
1092 }
1093 } else {
1094 level--;
1095 if (level < last_loop_level) {
1096 last_loop_opline = NULL;
1097 }
1098 TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
1099 }
1100 }
1101 #ifdef HAVE_GCC_GLOBAL_REGS
1102 prev_execute_data = execute_data;
1103 #endif
1104 }
1105 if (EX(call) != prev_call) {
1106 if (EX(call)
1107 && EX(call)->prev_execute_data == prev_call) {
1108 zend_function *func;
1109 uint32_t info = 0;
1110 zend_jit_op_array_trace_extension *jit_extension;
1111
1112 func = EX(call)->func;
1113 if (func->type == ZEND_INTERNAL_FUNCTION
1114 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
1115 func = NULL;
1116 } else if (func->type == ZEND_USER_FUNCTION) {
1117 jit_extension =
1118 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
1119 if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE))
1120 || (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))
1121 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) {
1122 func = NULL;
1123 } else if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) {
1124 func = (zend_function*)jit_extension->op_array;
1125 }
1126 }
1127
1128 #ifndef HAVE_GCC_GLOBAL_REGS
1129 opline = EX(opline);
1130 #endif
1131
1132 if (!func
1133 || (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)
1134 || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE)
1135 || func->common.prop_info) {
1136 /* continue recording */
1137 func = NULL;
1138 } else if (JIT_G(max_polymorphic_calls) == 0
1139 && zend_jit_may_be_polymorphic_call(opline - 1)) {
1140 func = NULL;
1141 ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC);
1142 } else if ((is_megamorphic == ZEND_JIT_EXIT_METHOD_CALL
1143 || is_megamorphic == ZEND_JIT_EXIT_CLOSURE_CALL)
1144 && trace_buffer[1].opline == opline - 1) {
1145 func = NULL;
1146 ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC);
1147 }
1148 if (!func) {
1149 info = ZEND_JIT_TRACE_NUM_ARGS_INFO(ZEND_CALL_NUM_ARGS(EX(call)));
1150 }
1151 TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, info, func);
1152 }
1153 prev_call = EX(call);
1154 }
1155
1156 #ifndef HAVE_GCC_GLOBAL_REGS
1157 opline = EX(opline);
1158 #endif
1159
1160 if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
1161 /* Abort trace because of exception */
1162 stop = ZEND_JIT_TRACE_STOP_EXCEPTION;
1163 break;
1164 }
1165
1166 trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
1167 if (trace_flags) {
1168 if (trace_flags & ZEND_JIT_TRACE_JITED) {
1169 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
1170 if ((start & ZEND_JIT_TRACE_START_LOOP) != 0
1171 && level + ret_level == 0
1172 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1173 ZEND_JIT_TRACE_STOP_COMPILED_LOOP) {
1174 /* Fail to try close outer loop through side exit.
1175 If this doesn't work just link. */
1176 stop = ZEND_JIT_TRACE_STOP_COMPILED_LOOP;
1177 break;
1178 } else {
1179 stop = ZEND_JIT_TRACE_STOP_LINK;
1180 break;
1181 }
1182 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
1183 if (start != ZEND_JIT_TRACE_START_RETURN) {
1184 // TODO: We may try to inline function ???
1185 stop = ZEND_JIT_TRACE_STOP_LINK;
1186 break;
1187 }
1188 if (backtrack_link_to_enter < 0) {
1189 backtrack_link_to_enter = idx;
1190 link_to_enter_opline = opline;
1191 }
1192 } else {
1193 stop = ZEND_JIT_TRACE_STOP_LINK;
1194 break;
1195 }
1196 } else if (trace_flags & ZEND_JIT_TRACE_BLACKLISTED) {
1197 stop = ZEND_JIT_TRACE_STOP_BLACK_LIST;
1198 break;
1199 } else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
1200 uint8_t bad_stop;
1201
1202 if (start != ZEND_JIT_TRACE_START_SIDE) {
1203 if (opline == orig_opline && level + ret_level == 0) {
1204 stop = ZEND_JIT_TRACE_STOP_LOOP;
1205 break;
1206 }
1207 }
1208
1209 if (start != ZEND_JIT_TRACE_START_SIDE
1210 || level + ret_level != 0) {
1211 /* First try creating a trace for inner loop.
1212 If this doesn't work try loop unroling. */
1213 bad_stop = zend_jit_trace_bad_stop_event(opline,
1214 JIT_G(blacklist_root_trace) / 2);
1215 if (bad_stop != ZEND_JIT_TRACE_STOP_INNER_LOOP
1216 && bad_stop != ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
1217 if (start == ZEND_JIT_TRACE_START_SIDE
1218 || zend_jit_trace_bad_stop_event(orig_opline,
1219 JIT_G(blacklist_root_trace) / 2) != ZEND_JIT_TRACE_STOP_INNER_LOOP) {
1220 stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
1221 break;
1222 }
1223 }
1224 }
1225
1226 if (opline == last_loop_opline
1227 && level == last_loop_level) {
1228 idx = zend_jit_trace_subtrace(trace_buffer,
1229 last_loop, idx, ZEND_JIT_TRACE_START_LOOP, op_array, opline);
1230 start = ZEND_JIT_TRACE_START_LOOP;
1231 stop = ZEND_JIT_TRACE_STOP_LOOP;
1232 ret_level = 0;
1233 break;
1234 } else if (loop_unroll_limit < JIT_G(max_loop_unrolls)) {
1235 last_loop = idx;
1236 last_loop_opline = opline;
1237 last_loop_level = level;
1238 loop_unroll_limit++;
1239 } else {
1240 stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL;
1241 break;
1242 }
1243 } else if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
1244 TRACE_RECORD(ZEND_JIT_TRACE_VM, 0, opline);
1245 stop = ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
1246 break;
1247 }
1248 }
1249 }
1250
1251 end_opline = opline;
1252 if (!ZEND_JIT_TRACE_STOP_OK(stop)) {
1253 if (backtrack_recursion > 0) {
1254 idx = backtrack_recursion;
1255 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
1256 end_opline = orig_opline;
1257 } else if (backtrack_ret_recursion > 0) {
1258 idx = backtrack_ret_recursion;
1259 ret_level = backtrack_ret_recursion_level;
1260 stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
1261 end_opline = orig_opline;
1262 } else if (backtrack_link_to_enter > 0) {
1263 if (stop == ZEND_JIT_TRACE_STOP_DEEP_RECURSION
1264 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) / 2) ==
1265 ZEND_JIT_TRACE_STOP_DEEP_RECURSION) {
1266 idx = backtrack_link_to_enter;
1267 stop = ZEND_JIT_TRACE_STOP_LINK;
1268 end_opline = link_to_enter_opline;
1269 }
1270 }
1271 }
1272
1273 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
1274 /* Shrink fake INIT_CALLs */
1275 while (trace_buffer[idx-1].op == ZEND_JIT_TRACE_INIT_CALL
1276 && (trace_buffer[idx-1].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1277 idx--;
1278 }
1279 }
1280
1281 TRACE_END(ZEND_JIT_TRACE_END, stop, end_opline);
1282
1283 #ifdef HAVE_GCC_GLOBAL_REGS
1284 if (!halt) {
1285 EX(opline) = opline;
1286 }
1287 #endif
1288
1289 #ifdef HAVE_GCC_GLOBAL_REGS
1290 execute_data = save_execute_data;
1291 opline = save_opline;
1292 #endif
1293
1294 return stop | halt;
1295 }
1296
