1 /*
2    +----------------------------------------------------------------------+
3    | Zend JIT                                                             |
4    +----------------------------------------------------------------------+
5    | Copyright (c) The PHP Group                                          |
6    +----------------------------------------------------------------------+
7    | This source file is subject to version 3.01 of the PHP license,      |
8    | that is bundled with this package in the file LICENSE, and is        |
9    | available through the world-wide-web at the following url:           |
10    | https://www.php.net/license/3_01.txt                                 |
11    | If you did not receive a copy of the PHP license and are unable to   |
12    | obtain it through the world-wide-web, please send a note to          |
13    | license@php.net so we can mail you a copy immediately.               |
14    +----------------------------------------------------------------------+
15    | Authors: Dmitry Stogov <dmitry@php.net>                              |
16    |          Xinchen Hui <laruence@php.net>                              |
17    +----------------------------------------------------------------------+
18 */
19 
20 #include "Zend/zend_execute.h"
21 #include "Zend/zend_exceptions.h"
22 #include "Zend/zend_vm.h"
23 #include "Zend/zend_closures.h"
24 #include "Zend/zend_constants.h"
25 #include "Zend/zend_API.h"
26 
27 #include <ZendAccelerator.h>
28 #include "Optimizer/zend_func_info.h"
29 #include "Optimizer/zend_call_graph.h"
30 #include "zend_jit.h"
31 #if ZEND_JIT_TARGET_X86
32 # include "zend_jit_x86.h"
33 #elif ZEND_JIT_TARGET_ARM64
34 # include "zend_jit_arm64.h"
35 #endif
36 
37 #include "zend_jit_internal.h"
38 
39 #ifdef HAVE_GCC_GLOBAL_REGS
40 # pragma GCC diagnostic ignored "-Wvolatile-register-var"
41 # if defined(__x86_64__)
42 register zend_execute_data* volatile execute_data __asm__("%r14");
43 register const zend_op* volatile opline __asm__("%r15");
44 # elif defined(i386)
45 register zend_execute_data* volatile execute_data __asm__("%esi");
46 register const zend_op* volatile opline __asm__("%edi");
47 # elif defined(__aarch64__)
48 register zend_execute_data* volatile execute_data __asm__("x27");
49 register const zend_op* volatile opline __asm__("x28");
50 # endif
51 # pragma GCC diagnostic warning "-Wvolatile-register-var"
52 #endif
53 
zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)54 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC)
55 {
56 	zend_execute_data *old_execute_data;
57 
58 	if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
59 		zend_clean_and_cache_symbol_table(EX(symbol_table));
60 	}
61 
62 	zend_vm_stack_free_extra_args_ex(call_info, execute_data);
63 	if (UNEXPECTED(call_info & ZEND_CALL_RELEASE_THIS)) {
64 		OBJ_RELEASE(Z_OBJ(execute_data->This));
65 	} else if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
66 		OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
67 	}
68 	if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
69 		zend_free_extra_named_params(EX(extra_named_params));
70 	}
71 
72 	old_execute_data = execute_data;
73 	execute_data = EX(prev_execute_data);
74 	zend_vm_stack_free_call_frame_ex(call_info, old_execute_data);
75 
76 	if (UNEXPECTED(EG(exception) != NULL)) {
77 		const zend_op *old_opline = EX(opline);
78 		zend_throw_exception_internal(NULL);
79 		if (old_opline->result_type != IS_UNDEF) {
80 			zval_ptr_dtor(EX_VAR(old_opline->result.var));
81 		}
82 #ifndef HAVE_GCC_GLOBAL_REGS
83 		return 2; // ZEND_VM_LEAVE
84 #endif
85 	} else {
86 		EX(opline)++;
87 #ifdef HAVE_GCC_GLOBAL_REGS
88 		opline = EX(opline);
89 #else
90 		return 2; // ZEND_VM_LEAVE
91 #endif
92 	}
93 }
94 
zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)95 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC)
96 {
97 	if (UNEXPECTED(call_info & (ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS))) {
98 		if (UNEXPECTED(call_info & ZEND_CALL_HAS_SYMBOL_TABLE)) {
99 			zend_clean_and_cache_symbol_table(EX(symbol_table));
100 		}
101 		zend_vm_stack_free_extra_args_ex(call_info, execute_data);
102 	}
103 	if (UNEXPECTED(call_info & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) {
104 		zend_free_extra_named_params(EX(extra_named_params));
105 	}
106 	if (UNEXPECTED(call_info & ZEND_CALL_CLOSURE)) {
107 		OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
108 	}
109 	execute_data = EG(current_execute_data);
110 #ifdef HAVE_GCC_GLOBAL_REGS
111 	opline = zend_jit_halt_op;
112 #else
113 	return -1; // ZEND_VM_RETURN
114 #endif
115 }
116 
zend_jit_leave_func_helper(EXECUTE_DATA_D)117 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(EXECUTE_DATA_D)
118 {
119 	uint32_t call_info = EX_CALL_INFO();
120 
121 	if (call_info & ZEND_CALL_TOP) {
122 		ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_top_func_helper, call_info);
123 	} else {
124 		ZEND_OPCODE_TAIL_CALL_EX(zend_jit_leave_nested_func_helper, call_info);
125 	}
126 }
127 
zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)128 void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D)
129 {
130 	zend_op_array *op_array = &EX(func)->op_array;
131 
132 	if (EXPECTED(!(op_array->fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))) {
133 		uint32_t first_extra_arg = op_array->num_args;
134 		uint32_t num_args = EX_NUM_ARGS();
135 		zval *end, *src, *dst;
136 		uint32_t type_flags = 0;
137 
138 		if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) {
139 			/* Skip useless ZEND_RECV and ZEND_RECV_INIT opcodes */
140 #ifdef HAVE_GCC_GLOBAL_REGS
141 			opline += first_extra_arg;
142 #else
143 			EX(opline) += first_extra_arg;
144 #endif
145 		}
146 
147 		/* move extra args into separate array after all CV and TMP vars */
148 		end = EX_VAR_NUM(first_extra_arg - 1);
149 		src = end + (num_args - first_extra_arg);
150 		dst = src + (op_array->last_var + op_array->T - first_extra_arg);
151 		if (EXPECTED(src != dst)) {
152 			do {
153 				type_flags |= Z_TYPE_INFO_P(src);
154 				ZVAL_COPY_VALUE(dst, src);
155 				ZVAL_UNDEF(src);
156 				src--;
157 				dst--;
158 			} while (src != end);
159 			if (type_flags & (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT)) {
160 				ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
161 			}
162 		} else {
163 			do {
164 				if (Z_REFCOUNTED_P(src)) {
165 					ZEND_ADD_CALL_FLAG(execute_data, ZEND_CALL_FREE_EXTRA_ARGS);
166 					break;
167 				}
168 				src--;
169 			} while (src != end);
170 		}
171 	}
172 }
173 
zend_jit_deprecated_helper(OPLINE_D)174 bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D)
175 {
176 	zend_execute_data *call = (zend_execute_data *) opline;
177 	zend_function *fbc = call->func;
178 
179 	zend_deprecated_function(fbc);
180 
181 	if (EG(exception)) {
182 #ifndef HAVE_GCC_GLOBAL_REGS
183 		zend_execute_data *execute_data = EG(current_execute_data);
184 #endif
185 		const zend_op *opline = EG(opline_before_exception);
186 		if (opline && RETURN_VALUE_USED(opline)) {
187 			ZVAL_UNDEF(EX_VAR(opline->result.var));
188 		}
189 
190 		zend_vm_stack_free_args(call);
191 
192 		if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) {
193 			OBJ_RELEASE(Z_OBJ(call->This));
194 		}
195 
196 		zend_vm_stack_free_call_frame(call);
197 		return 0;
198 	}
199 	return 1;
200 }
201 
zend_jit_undefined_long_key(EXECUTE_DATA_D)202 void ZEND_FASTCALL zend_jit_undefined_long_key(EXECUTE_DATA_D)
203 {
204 	const zend_op *opline = EX(opline);
205 	zval *result = EX_VAR(opline->result.var);
206 	zval *dim;
207 
208 	if (opline->op2_type == IS_CONST) {
209 		dim = RT_CONSTANT(opline, opline->op2);
210 	} else {
211 		dim = EX_VAR(opline->op2.var);
212 	}
213 	ZEND_ASSERT(Z_TYPE_P(dim) == IS_LONG);
214 	zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, Z_LVAL_P(dim));
215 	ZVAL_NULL(result);
216 }
217 
zend_jit_undefined_string_key(EXECUTE_DATA_D)218 void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D)
219 {
220 	const zend_op *opline = EX(opline);
221 	zval *result = EX_VAR(opline->result.var);
222 	zval *dim;
223 	zend_ulong lval;
224 
225 	if (opline->op2_type == IS_CONST) {
226 		dim = RT_CONSTANT(opline, opline->op2);
227 	} else {
228 		dim = EX_VAR(opline->op2.var);
229 	}
230 	ZEND_ASSERT(Z_TYPE_P(dim) == IS_STRING);
231 	if (ZEND_HANDLE_NUMERIC(Z_STR_P(dim), lval)) {
232 		zend_error(E_WARNING, "Undefined array key " ZEND_LONG_FMT, lval);
233 	} else {
234 		zend_error(E_WARNING, "Undefined array key \"%s\"", Z_STRVAL_P(dim));
235 	}
236 	ZVAL_NULL(result);
237 }
238 
zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)239 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS)
240 {
241 	zend_op_array *op_array = (zend_op_array*)EX(func);
242 	zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
243 	zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t) jit_extension->orig_handler;
244 	++*(uintptr_t*)(EX(run_time_cache) + zend_jit_profile_counter_rid);
245 	++zend_jit_profile_counter;
246 	ZEND_OPCODE_TAIL_CALL(handler);
247 }
248 
zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)249 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
250 {
251 	zend_jit_op_array_hot_extension *jit_extension =
252 		(zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
253 #ifndef HAVE_GCC_GLOBAL_REGS
254 	const zend_op *opline = EX(opline);
255 #endif
256 
257 	*(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func));
258 
259 	if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
260 		*(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
261 		zend_jit_hot_func(execute_data, opline);
262 		ZEND_OPCODE_RETURN();
263 	} else {
264 		zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
265 		ZEND_OPCODE_TAIL_CALL(handler);
266 	}
267 }
268 
zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)269 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
270 {
271 	zend_jit_op_array_hot_extension *jit_extension =
272 		(zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
273 #ifndef HAVE_GCC_GLOBAL_REGS
274 	const zend_op *opline = EX(opline);
275 #endif
276 
277 	*(jit_extension->counter) -= ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop));
278 
279 	if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
280 		*(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
281 		zend_jit_hot_func(execute_data, opline);
282 		ZEND_OPCODE_RETURN();
283 	} else {
284 		zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
285 		ZEND_OPCODE_TAIL_CALL(handler);
286 	}
287 }
288 
_zend_quick_get_constant(const zval * key,uint32_t flags,int check_defined_only)289 static zend_always_inline zend_constant* _zend_quick_get_constant(
290 		const zval *key, uint32_t flags, int check_defined_only)
291 {
292 #ifndef HAVE_GCC_GLOBAL_REGS
293 	zend_execute_data *execute_data = EG(current_execute_data);
294 #endif
295 	const zend_op *opline = EX(opline);
296 	zval *zv;
297 	zend_constant *c = NULL;
298 
299 	/* null/true/false are resolved during compilation, so don't check for them here. */
300 	zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
301 	if (zv) {
302 		c = (zend_constant*)Z_PTR_P(zv);
303 	} else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
304 		key++;
305 		zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
306 		if (zv) {
307 			c = (zend_constant*)Z_PTR_P(zv);
308 		}
309 	}
310 
311 	if (!c) {
312 		if (!check_defined_only) {
313 			zend_throw_error(NULL, "Undefined constant \"%s\"", Z_STRVAL_P(RT_CONSTANT(opline, opline->op2)));
314 			ZVAL_UNDEF(EX_VAR(opline->result.var));
315 		}
316 		CACHE_PTR(opline->extended_value, ENCODE_SPECIAL_CACHE_NUM(zend_hash_num_elements(EG(zend_constants))));
317 		return NULL;
318 	}
319 
320 	if (!check_defined_only) {
321 		if (ZEND_CONSTANT_FLAGS(c) & CONST_DEPRECATED) {
322 			zend_error(E_DEPRECATED, "Constant %s is deprecated", ZSTR_VAL(c->name));
323 			if (EG(exception)) {
324 				return NULL;
325 			}
326 			return c;
327 		}
328 	}
329 
330 	CACHE_PTR(opline->extended_value, c);
331 	return c;
332 }
333 
zend_jit_get_constant(const zval * key,uint32_t flags)334 zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags)
335 {
336 	return _zend_quick_get_constant(key, flags, 0);
337 }
338 
zend_jit_check_constant(const zval * key)339 zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key)
340 {
341 	return _zend_quick_get_constant(key, 0, 1);
342 }
343 
zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)344 static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(uint32_t cost ZEND_OPCODE_HANDLER_ARGS_DC)
345 {
346 	zend_jit_op_array_trace_extension *jit_extension =
347 		(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
348 	size_t offset = jit_extension->offset;
349 #ifndef HAVE_GCC_GLOBAL_REGS
350 	const zend_op *opline = EX(opline);
351 #endif
352 
353 	*(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
354 
355 	if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
356 		*(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT;
357 		if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
358 #ifdef HAVE_GCC_GLOBAL_REGS
359 			opline = NULL;
360 			return;
361 #else
362 			return -1;
363 #endif
364 		}
365 #ifdef HAVE_GCC_GLOBAL_REGS
366 		execute_data = EG(current_execute_data);
367 		opline = execute_data ? EX(opline) : NULL;
368 		return;
369 #else
370 		return 1;
371 #endif
372 	} else {
373 		zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
374 		ZEND_OPCODE_TAIL_CALL(handler);
375 	}
376 }
377 
zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)378 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
379 {
380 	ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
381 		((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
382 }
383 
zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)384 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
385 {
386 	ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
387 		((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
388 }
389 
zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)390 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
391 {
392 	ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper,
393 		((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
394 }
395 
396 #define TRACE_RECORD(_op, _info, _ptr) \
397 	trace_buffer[idx].info = _op | (_info); \
398 	trace_buffer[idx].ptr = _ptr; \
399 	idx++; \
400 	if (idx >= JIT_G(max_trace_length) - 2) { \
401 		stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
402 		break; \
403 	}
404 
405 #define TRACE_RECORD_VM(_op, _ptr, _op1_type, _op2_type, _op3_type) \
406 	trace_buffer[idx].op = _op; \
407 	trace_buffer[idx].op1_type = _op1_type; \
408 	trace_buffer[idx].op2_type = _op2_type; \
409 	trace_buffer[idx].op3_type = _op3_type; \
410 	trace_buffer[idx].ptr = _ptr; \
411 	idx++; \
412 	if (idx >= JIT_G(max_trace_length) - 2) { \
413 		stop = ZEND_JIT_TRACE_STOP_TOO_LONG; \
414 		break; \
415 	}
416 
417 #define TRACE_START(_op, _start, _ptr1, _ptr2) \
418 	trace_buffer[0].op = _op; \
419 	trace_buffer[0].start = _start; \
420 	trace_buffer[0].level = 0; \
421 	trace_buffer[0].ptr = _ptr1; \
422 	trace_buffer[1].last = 0; \
423 	trace_buffer[1].ptr = _ptr2; \
424 	idx = ZEND_JIT_TRACE_START_REC_SIZE;
425 
426 #define TRACE_END(_op, _stop, _ptr) \
427 	trace_buffer[1].last = idx; \
428 	trace_buffer[idx].op   = _op; \
429 	trace_buffer[idx].start = trace_buffer[idx].start; \
430 	trace_buffer[idx].stop = trace_buffer[0].stop = _stop; \
431 	trace_buffer[idx].level = trace_buffer[0].level = ret_level ? ret_level + 1 : 0; \
432 	trace_buffer[idx].ptr  = _ptr;
433 
zend_jit_trace_recursive_call_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level,int level)434 static int zend_jit_trace_recursive_call_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level, int level)
435 {
436 	int i;
437 	int count = 0;
438 
439 	for (i = ret_level; i < level; i++) {
440 		count += (unrolled_calls[i] == op_array);
441 	}
442 	return count;
443 }
444 
zend_jit_trace_recursive_ret_count(const zend_op_array * op_array,const zend_op_array ** unrolled_calls,int ret_level)445 static int zend_jit_trace_recursive_ret_count(const zend_op_array *op_array, const zend_op_array **unrolled_calls, int ret_level)
446 {
447 	int i;
448 	int count = 0;
449 
450 	for (i = 0; i < ret_level; i++) {
451 		count += (unrolled_calls[i] == op_array);
452 	}
453 	return count;
454 }
455 
zend_jit_trace_has_recursive_ret(zend_execute_data * ex,const zend_op_array * orig_op_array,const zend_op * orig_opline,int ret_level)456 static int zend_jit_trace_has_recursive_ret(zend_execute_data *ex, const zend_op_array *orig_op_array, const zend_op *orig_opline, int ret_level)
457 {
458 	while (ex != NULL && ex->func != NULL && ret_level < ZEND_JIT_TRACE_MAX_RET_DEPTH) {
459 		if (&ex->func->op_array == orig_op_array && ex->opline + 1 == orig_opline) {
460 			return 1;
461 		}
462 		ex = ex->prev_execute_data;
463 		ret_level++;
464 	}
465 	return 0;
466 }
467 
zend_jit_trace_bad_stop_event(const zend_op * opline,int count)468 static uint8_t zend_jit_trace_bad_stop_event(const zend_op *opline, int count)
469 {
470 	const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
471 	uint8_t *cache_count = JIT_G(bad_root_cache_count);
472 	uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
473 	uint32_t i;
474 
475 	if (count < 0) {
476 		count = 0;
477 	}
478 	for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
479 		if (cache_opline[i] == opline) {
480 			if (cache_count[i] >= count) {
481 				return cache_stop[i];
482 			}
483 			break;
484 		}
485 	}
486 	return 0;
487 }
488 
489 #define ZEND_CALL_MEGAMORPHIC ZEND_CALL_JIT_RESERVED
490 
zend_jit_trace_record_fake_init_call_ex(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic,uint32_t init_level)491 static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic, uint32_t init_level)
492 {
493 	zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
494 
495 	do {
496 		zend_function *func;
497 		zend_jit_op_array_trace_extension *jit_extension;
498 
499 		if (call->prev_execute_data) {
500 			idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, init_level + 1);
501 			if (idx < 0) {
502 				return idx;
503 			}
504 		}
505 
506 		func = call->func;
507 		if (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE|ZEND_ACC_NEVER_CACHE)) {
508 			/* TODO: Can we continue recording ??? */
509 			return -1;
510 		}
511 		if (func->type == ZEND_INTERNAL_FUNCTION
512 		 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
513 			return -1;
514 		}
515 		if (func->type == ZEND_USER_FUNCTION
516 		 && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
517 			jit_extension =
518 				(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
519 			if (UNEXPECTED(!jit_extension
520 			 || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)
521 			 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE))) {
522 				return -1;
523 			}
524 			func = (zend_function*)jit_extension->op_array;
525 		}
526 		if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM
527 		 /* TODO: use more accurate check ??? */
528 		 && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
529 		  || func->common.scope)) {
530 			func = NULL;
531 			ZEND_ADD_CALL_FLAG(call, ZEND_CALL_MEGAMORPHIC);
532 		}
533 		TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INFO(init_level), func);
534 	} while (0);
535 	return idx;
536 }
537 
zend_jit_trace_record_fake_init_call(zend_execute_data * call,zend_jit_trace_rec * trace_buffer,int idx,uint32_t is_megamorphic)538 static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, uint32_t is_megamorphic)
539 {
540 	return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, 0);
541 }
542 
zend_jit_trace_subtrace(zend_jit_trace_rec * trace_buffer,int start,int end,uint8_t event,const zend_op_array * op_array,const zend_op * opline)543 static int zend_jit_trace_subtrace(zend_jit_trace_rec *trace_buffer, int start, int end, uint8_t event, const zend_op_array *op_array, const zend_op *opline)
544 {
545 	int idx;
546 
547 	TRACE_START(ZEND_JIT_TRACE_START, event, op_array, opline);
548 	memmove(trace_buffer + idx, trace_buffer + start, (end - start) * sizeof(zend_jit_trace_rec));
549 	return idx + (end - start);
550 }
551 
552 /*
553  *  Trace Linking Rules
554  *  ===================
555  *
556  *                                          flags
557  *          +----------+----------+----------++----------+----------+----------+
558  *          |                                ||              JIT               |
559  *          +----------+----------+----------++----------+----------+----------+
560  *   start  |   LOOP   |  ENTER   |  RETURN  ||   LOOP   |  ENTER   |  RETURN  |
561  * +========+==========+==========+==========++==========+==========+==========+
562  * | LOOP   |   loop   |          | loop-ret || COMPILED |   LINK   |   LINK   |
563  * +--------+----------+----------+----------++----------+----------+----------+
564  * | ENTER  |INNER_LOOP| rec-call |  return  ||   LINK   |   LINK   |   LINK   |
565  * +--------+----------+----------+----------++----------+----------+----------+
566  * | RETURN |INNER_LOOP|          |  rec-ret ||   LINK   |          |   LINK   |
567  * +--------+----------+----------+----------++----------+----------+----------+
568  * | SIDE   |  unroll  |          |  return  ||   LINK   |   LINK   |   LINK   |
569  * +--------+----------+----------+----------++----------+----------+----------+
570  *
571  * loop:       LOOP if "cycle" and level == 0, otherwise INNER_LOOP
572  * INNER_LOOP: abort recording and start new one (wait for loop)
573  * COMPILED:   abort recording (wait while side exit creates outer loop)
574  * unroll:     continue recording while unroll limit reached
575  * rec-call:   RECURSIVE_CALL if "cycle" and level > N, otherwise continue
576  * loop-ret:   LOOP_EXIT if level == 0, otherwise continue (wait for loop)
577  * return:     RETURN if level == 0
578  * rec_ret:    RECURSIVE_RET if "cycle" and ret_level > N, otherwise continue
579  *
580  */
581 
zend_jit_trace_execute(zend_execute_data * ex,const zend_op * op,zend_jit_trace_rec * trace_buffer,uint8_t start,uint32_t is_megamorphic)582 zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphic)
583 
584 {
585 #ifdef HAVE_GCC_GLOBAL_REGS
586 	zend_execute_data *save_execute_data = execute_data;
587 	const zend_op *save_opline = opline;
588 #endif
589 	const zend_op *orig_opline, *end_opline;
590 	zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
591 	zend_jit_trace_stop halt = 0;
592 	int level = 0;
593 	int ret_level = 0;
594 	zend_vm_opcode_handler_t handler;
595 	const zend_op_array *op_array;
596 	zend_jit_op_array_trace_extension *jit_extension;
597 	size_t offset;
598 	int idx, count;
599 	uint8_t  trace_flags, op1_type, op2_type, op3_type;
600 	zend_class_entry *ce1, *ce2;
601 	const zend_op *link_to_enter_opline = NULL;
602 	int backtrack_link_to_enter = -1;
603 	int backtrack_recursion = -1;
604 	int backtrack_ret_recursion = -1;
605 	int backtrack_ret_recursion_level = 0;
606 	int loop_unroll_limit = 0;
607 	int last_loop = -1;
608 	int last_loop_level = -1;
609 	const zend_op *last_loop_opline = NULL;
610 	const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
611 #ifdef HAVE_GCC_GLOBAL_REGS
612 	zend_execute_data *prev_execute_data = ex;
613 
614 	execute_data = ex;
615 	opline = EX(opline) = op;
616 #else
617 	int rc;
618 	zend_execute_data *execute_data = ex;
619 	const zend_op *opline = EX(opline);
620 #endif
621 	zend_execute_data *prev_call = EX(call);
622 
623 	orig_opline = opline;
624 
625 	op_array = &EX(func)->op_array;
626 	jit_extension =
627 		(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
628 	offset = jit_extension->offset;
629 	if (!op_array->function_name
630 	 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
631 		op_array = jit_extension->op_array;
632 	}
633 
634 	TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline);
635 
636 	if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
637 		/* Abort trace because of exception */
638 		TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_EXCEPTION, opline);
639 #ifdef HAVE_GCC_GLOBAL_REGS
640 		execute_data = save_execute_data;
641 		opline = save_opline;
642 #endif
643 		return ZEND_JIT_TRACE_STOP_EXCEPTION;
644 	}
645 
646 	trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
647 	if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
648 		TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_NOT_SUPPORTED, opline);
649 #ifdef HAVE_GCC_GLOBAL_REGS
650 		execute_data = save_execute_data;
651 		opline = save_opline;
652 #endif
653 		return ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
654 	}
655 
656 	if (prev_call) {
657 		int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic);
658 		if (ret < 0) {
659 			TRACE_END(ZEND_JIT_TRACE_END, ZEND_JIT_TRACE_STOP_BAD_FUNC, opline);
660 #ifdef HAVE_GCC_GLOBAL_REGS
661 			execute_data = save_execute_data;
662 			opline = save_opline;
663 #endif
664 			return ZEND_JIT_TRACE_STOP_BAD_FUNC;
665 		}
666 		idx = ret;
667 	}
668 
669 	while (1) {
670 		ce1 = ce2 = NULL;
671 		op1_type = op2_type = op3_type = IS_UNKNOWN;
672 		if ((opline->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV))
673 		 && opline->opcode != ZEND_ROPE_ADD
674 		 && opline->opcode != ZEND_ROPE_END
675 		 && opline->opcode != ZEND_NEW
676 		 && opline->opcode != ZEND_FETCH_CLASS_CONSTANT
677 		 && opline->opcode != ZEND_INIT_STATIC_METHOD_CALL) {
678 			zval *zv = EX_VAR(opline->op1.var);
679 			op1_type = Z_TYPE_P(zv);
680 			uint8_t flags = 0;
681 
682 			if (op1_type == IS_INDIRECT) {
683 				zv = Z_INDIRECT_P(zv);
684 				op1_type = Z_TYPE_P(zv);
685 				flags |= IS_TRACE_INDIRECT;
686 			}
687 			if (op1_type == IS_REFERENCE) {
688 				zv = Z_REFVAL_P(zv);
689 				op1_type = Z_TYPE_P(zv);
690 				flags |= IS_TRACE_REFERENCE;
691 			}
692 			if (Z_TYPE_P(zv) == IS_OBJECT) {
693 				ce1 = Z_OBJCE_P(zv);
694 			} else if (Z_TYPE_P(zv) == IS_ARRAY) {
695 				if (HT_IS_PACKED(Z_ARRVAL_P(zv))) {
696 					flags |= IS_TRACE_PACKED;
697 				}
698 			}
699 			op1_type |= flags;
700 		}
701 		if (opline->op2_type & (IS_TMP_VAR|IS_VAR|IS_CV)
702 		 && opline->opcode != ZEND_INSTANCEOF
703 		 && opline->opcode != ZEND_UNSET_STATIC_PROP
704 		 && opline->opcode != ZEND_ISSET_ISEMPTY_STATIC_PROP
705 		 && opline->opcode != ZEND_ASSIGN_STATIC_PROP
706 		 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_REF
707 		 && opline->opcode != ZEND_ASSIGN_STATIC_PROP_OP
708 		 && opline->opcode != ZEND_PRE_INC_STATIC_PROP
709 		 && opline->opcode != ZEND_POST_INC_STATIC_PROP
710 		 && opline->opcode != ZEND_PRE_DEC_STATIC_PROP
711 		 && opline->opcode != ZEND_POST_DEC_STATIC_PROP
712 		 && opline->opcode != ZEND_FETCH_STATIC_PROP_R
713 		 && opline->opcode != ZEND_FETCH_STATIC_PROP_W
714 		 && opline->opcode != ZEND_FETCH_STATIC_PROP_RW
715 		 && opline->opcode != ZEND_FETCH_STATIC_PROP_IS
716 		 && opline->opcode != ZEND_FETCH_STATIC_PROP_FUNC_ARG
717 		 && opline->opcode != ZEND_FETCH_STATIC_PROP_UNSET
718 		 && (opline->op2_type == IS_CV
719 		  || (opline->opcode != ZEND_FE_FETCH_R
720 		   && opline->opcode != ZEND_FE_FETCH_RW))) {
721 			zval *zv = EX_VAR(opline->op2.var);
722 			uint8_t flags = 0;
723 
724 			op2_type = Z_TYPE_P(zv);
725 			if (op2_type == IS_INDIRECT) {
726 				zv = Z_INDIRECT_P(zv);
727 				op2_type = Z_TYPE_P(zv);
728 				flags |= IS_TRACE_INDIRECT;
729 			}
730 			if (op2_type == IS_REFERENCE) {
731 				zv = Z_REFVAL_P(zv);
732 				op2_type = Z_TYPE_P(zv);
733 				flags |= IS_TRACE_REFERENCE;
734 			}
735 			if (Z_TYPE_P(zv) == IS_OBJECT) {
736 				ce2 = Z_OBJCE_P(zv);
737 			}
738 			op2_type |= flags;
739 		}
740 		if (opline->opcode == ZEND_ASSIGN_DIM ||
741 			opline->opcode == ZEND_ASSIGN_OBJ ||
742 			opline->opcode == ZEND_ASSIGN_STATIC_PROP ||
743 			opline->opcode == ZEND_ASSIGN_DIM_OP ||
744 			opline->opcode == ZEND_ASSIGN_OBJ_OP ||
745 			opline->opcode == ZEND_ASSIGN_STATIC_PROP_OP ||
746 			opline->opcode == ZEND_ASSIGN_OBJ_REF ||
747 			opline->opcode == ZEND_ASSIGN_STATIC_PROP_REF) {
748 			if ((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR|IS_CV)) {
749 				zval *zv = EX_VAR((opline+1)->op1.var);
750 				uint8_t flags = 0;
751 
752 				op3_type = Z_TYPE_P(zv);
753 				if (op3_type == IS_INDIRECT) {
754 					zv = Z_INDIRECT_P(zv);
755 					op3_type = Z_TYPE_P(zv);
756 					flags |= IS_TRACE_INDIRECT;
757 				}
758 				if (op3_type == IS_REFERENCE) {
759 					zv = Z_REFVAL_P(zv);
760 					op3_type = Z_TYPE_P(zv);
761 					flags |= IS_TRACE_REFERENCE;
762 				}
763 				op3_type |= flags;
764 			}
765 		}
766 
767 		TRACE_RECORD_VM(ZEND_JIT_TRACE_VM, opline, op1_type, op2_type, op3_type);
768 
769 		if (ce1) {
770 			TRACE_RECORD(ZEND_JIT_TRACE_OP1_TYPE, 0, ce1);
771 		}
772 
773 		if (ce2) {
774 			TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2);
775 		}
776 
777 		switch (opline->opcode) {
778 			case ZEND_FETCH_DIM_R:
779 			case ZEND_FETCH_DIM_W:
780 			case ZEND_FETCH_DIM_RW:
781 			case ZEND_FETCH_DIM_IS:
782 			case ZEND_FETCH_DIM_FUNC_ARG:
783 			case ZEND_FETCH_DIM_UNSET:
784 			case ZEND_FETCH_LIST_R:
785 			case ZEND_FETCH_LIST_W:
786 			case ZEND_ASSIGN_DIM:
787 			case ZEND_ASSIGN_DIM_OP:
788 			case ZEND_UNSET_DIM:
789 			case ZEND_ISSET_ISEMPTY_DIM_OBJ:
790 				if (opline->op1_type == IS_CONST) {
791 					zval *arr = RT_CONSTANT(opline, opline->op1);
792 					op1_type = Z_TYPE_P(arr);
793 				}
794 				if ((op1_type & IS_TRACE_TYPE_MASK) == IS_ARRAY
795 				 && opline->op2_type != IS_UNDEF) {
796 					zval *arr, *dim, *val;
797 					uint8_t val_type = IS_UNDEF;
798 
799 					if (opline->op2_type == IS_CONST) {
800 						dim	= RT_CONSTANT(opline, opline->op2);
801 					} else {
802 						dim = EX_VAR(opline->op2.var);
803 					}
804 
805 					if (Z_TYPE_P(dim) == IS_LONG || Z_TYPE_P(dim) == IS_STRING) {
806 						if (opline->op1_type == IS_CONST) {
807 							arr = RT_CONSTANT(opline, opline->op1);
808 						} else {
809 							arr = EX_VAR(opline->op1.var);
810 						}
811 						if (Z_TYPE_P(arr) == IS_INDIRECT) {
812 							arr = Z_INDIRECT_P(arr);
813 						}
814 						if (Z_TYPE_P(arr) == IS_REFERENCE) {
815 							arr = Z_REFVAL_P(arr);
816 						}
817 						ZEND_ASSERT(Z_TYPE_P(arr) == IS_ARRAY);
818 						if (Z_TYPE_P(dim) == IS_LONG) {
819 							val = zend_hash_index_find(Z_ARRVAL_P(arr), Z_LVAL_P(dim));
820 						} else /*if Z_TYPE_P(dim) == IS_STRING)*/ {
821 							val = zend_symtable_find(Z_ARRVAL_P(arr), Z_STR_P(dim));
822 						}
823 						if (val) {
824 							val_type = Z_TYPE_P(val);
825 						}
826 						TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, val_type, 0, 0);
827 					}
828 				}
829 				break;
830 			case ZEND_FETCH_OBJ_R:
831 			case ZEND_FETCH_OBJ_W:
832 			case ZEND_FETCH_OBJ_RW:
833 			case ZEND_FETCH_OBJ_IS:
834 			case ZEND_FETCH_OBJ_FUNC_ARG:
835 			case ZEND_FETCH_OBJ_UNSET:
836 			case ZEND_ASSIGN_OBJ:
837 			case ZEND_ASSIGN_OBJ_OP:
838 			case ZEND_ASSIGN_OBJ_REF:
839 			case ZEND_UNSET_OBJ:
840 			case ZEND_ISSET_ISEMPTY_PROP_OBJ:
841 			case ZEND_PRE_INC_OBJ:
842 			case ZEND_PRE_DEC_OBJ:
843 			case ZEND_POST_INC_OBJ:
844 			case ZEND_POST_DEC_OBJ:
845 				if (opline->op1_type != IS_CONST
846 				 && opline->op2_type == IS_CONST
847 				 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
848 				 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
849 					zval *obj, *val;
850 					zend_string *prop_name = Z_STR_P(RT_CONSTANT(opline, opline->op2));
851 					zend_property_info *prop_info;
852 
853 					if (opline->op1_type == IS_UNUSED) {
854 						obj = &EX(This);
855 					} else {
856 						obj = EX_VAR(opline->op1.var);
857 					}
858 					if (Z_TYPE_P(obj) != IS_OBJECT
859 					 || Z_OBJ_P(obj)->handlers != &std_object_handlers) {
860 						break;
861 					}
862 					prop_info = zend_get_property_info(Z_OBJCE_P(obj), prop_name, 1);
863 					if (prop_info
864 					 && prop_info != ZEND_WRONG_PROPERTY_INFO
865 					 && !(prop_info->flags & ZEND_ACC_STATIC)) {
866 						val = OBJ_PROP(Z_OBJ_P(obj), prop_info->offset);
867 						TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, Z_TYPE_P(val), 0, 0);
868 					}
869 				}
870 				break;
871 			default:
872 				break;
873 		}
874 
875 		if (opline->opcode == ZEND_DO_FCALL
876 		 || opline->opcode == ZEND_DO_ICALL
877 		 || opline->opcode == ZEND_DO_UCALL
878 		 ||	opline->opcode == ZEND_DO_FCALL_BY_NAME) {
879 			if (ZEND_CALL_INFO(EX(call)) & ZEND_CALL_MEGAMORPHIC) {
880 				stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
881 				break;
882 			}
883 			if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
884 				if (EX(call)->func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) {
885 					stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
886 					break;
887 				}
888 				TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func);
889 			}
890 		} else if (opline->opcode == ZEND_INCLUDE_OR_EVAL
891 				|| opline->opcode == ZEND_CALLABLE_CONVERT) {
892 			/* TODO: Support tracing JIT for ZEND_CALLABLE_CONVERT. */
893 			stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
894 			break;
895 		}
896 
897 		handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
898 #ifdef HAVE_GCC_GLOBAL_REGS
899 		handler();
900 		if (UNEXPECTED(opline == zend_jit_halt_op)) {
901 			stop = ZEND_JIT_TRACE_STOP_RETURN;
902 			opline = NULL;
903 			halt = ZEND_JIT_TRACE_HALT;
904 			break;
905 		}
906 		if (UNEXPECTED(execute_data != prev_execute_data)) {
907 #else
908 		rc = handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU);
909 		if (rc != 0) {
910 			if (rc < 0) {
911 				stop = ZEND_JIT_TRACE_STOP_RETURN;
912 				opline = NULL;
913 				halt = ZEND_JIT_TRACE_HALT;
914 				break;
915 			} else if (execute_data == EG(current_execute_data)) {
916 				/* return after interrupt handler */
917 				rc = 0;
918 			}
919 			execute_data = EG(current_execute_data);
920 			opline = EX(opline);
921 #endif
922 
923 			op_array = &EX(func)->op_array;
924 			jit_extension =
925 				(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
926 			if (UNEXPECTED(!jit_extension)
927 			 || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) {
928 				stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
929 				break;
930 			}
931 			offset = jit_extension->offset;
932 			if (!op_array->function_name
933 			 || (op_array->fn_flags & ZEND_ACC_CLOSURE)) {
934 				op_array = jit_extension->op_array;
935 			}
936 
937 #ifdef HAVE_GCC_GLOBAL_REGS
938 			if (execute_data->prev_execute_data == prev_execute_data) {
939 #else
940 			if (rc == 0) {
941 				/* pass */
942 			} else if (rc == 1) {
943 #endif
944 				/* Enter into function */
945 				prev_call = NULL;
946 				if (level > ZEND_JIT_TRACE_MAX_CALL_DEPTH) {
947 					stop = ZEND_JIT_TRACE_STOP_TOO_DEEP;
948 					break;
949 				}
950 
951 				if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
952 					/* TODO: Can we continue recording ??? */
953 					stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
954 					break;
955 				}
956 
957 				TRACE_RECORD(ZEND_JIT_TRACE_ENTER,
958 					EX(return_value) != NULL ? ZEND_JIT_TRACE_RETURN_VALUE_USED : 0,
959 					op_array);
960 
961 				count = zend_jit_trace_recursive_call_count(&EX(func)->op_array, unrolled_calls, ret_level, level);
962 
963 				if (opline == orig_opline) {
964 					if (count + 1 >= JIT_G(max_recursive_calls)) {
965 						stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
966 						break;
967 					}
968 					backtrack_recursion = idx;
969 				} else if (count >= JIT_G(max_recursive_calls)) {
970 					stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
971 					break;
972 				}
973 
974 				unrolled_calls[ret_level + level] = &EX(func)->op_array;
975 				level++;
976 			} else {
977 				/* Return from function */
978 				prev_call = EX(call);
979 				if (level == 0) {
980 					if (start == ZEND_JIT_TRACE_START_RETURN
981 					        && JIT_G(max_recursive_returns) > 0
982 					        && execute_data->prev_execute_data
983 					        && execute_data->prev_execute_data->func
984 					        && execute_data->prev_execute_data->func->type == ZEND_USER_FUNCTION
985 					        && zend_jit_trace_has_recursive_ret(execute_data, trace_buffer[0].op_array, orig_opline, ret_level)) {
986 						if (ret_level > ZEND_JIT_TRACE_MAX_RET_DEPTH) {
987 							stop = ZEND_JIT_TRACE_STOP_TOO_DEEP_RET;
988 							break;
989 						}
990 						TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
991 						count = zend_jit_trace_recursive_ret_count(&EX(func)->op_array, unrolled_calls, ret_level);
992 						if (opline == orig_opline) {
993 							if (count + 1 >= JIT_G(max_recursive_returns)) {
994 								stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
995 								break;
996 							}
997 							backtrack_ret_recursion = idx;
998 							backtrack_ret_recursion_level = ret_level;
999 						} else if (count >= JIT_G(max_recursive_returns)) {
1000 							stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
1001 							break;
1002 						}
1003 
1004 						unrolled_calls[ret_level] = &EX(func)->op_array;
1005 						ret_level++;
1006 						last_loop_opline = NULL;
1007 
1008 						if (prev_call) {
1009 							int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0);
1010 							if (ret < 0) {
1011 								stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1012 								break;
1013 							}
1014 							idx = ret;
1015 						}
1016 					} else if (start & ZEND_JIT_TRACE_START_LOOP
1017 					 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1018 							ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
1019 						/* Fail to try close the loop.
1020 						   If this doesn't work terminate it. */
1021 						stop = ZEND_JIT_TRACE_STOP_LOOP_EXIT;
1022 						break;
1023 					} else if (start & ZEND_JIT_TRACE_START_ENTER
1024 					 && EX(prev_execute_data)
1025 					 && EX(func) == EX(prev_execute_data)->func
1026 					 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1027 							ZEND_JIT_TRACE_STOP_RECURSION_EXIT) {
1028 						stop = ZEND_JIT_TRACE_STOP_RECURSION_EXIT;
1029 						break;
1030 					} else {
1031 						stop = ZEND_JIT_TRACE_STOP_RETURN;
1032 						break;
1033 					}
1034 				} else {
1035 					level--;
1036 					if (level < last_loop_level) {
1037 						last_loop_opline = NULL;
1038 					}
1039 					TRACE_RECORD(ZEND_JIT_TRACE_BACK, 0, op_array);
1040 				}
1041 			}
1042 #ifdef HAVE_GCC_GLOBAL_REGS
1043 			prev_execute_data = execute_data;
1044 #endif
1045 		}
1046 		if (EX(call) != prev_call) {
1047 			if (EX(call)
1048 			 && EX(call)->prev_execute_data == prev_call) {
1049 				zend_function *func;
1050 				zend_jit_op_array_trace_extension *jit_extension;
1051 
1052 				if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) {
1053 					/* TODO: Can we continue recording ??? */
1054 					stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE;
1055 					break;
1056 				} else if (EX(call)->func->common.fn_flags & ZEND_ACC_NEVER_CACHE) {
1057 					/* TODO: Can we continue recording ??? */
1058 					stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1059 					break;
1060 				}
1061 				func = EX(call)->func;
1062 				if (func->type == ZEND_INTERNAL_FUNCTION
1063 				 && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) {
1064 					stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
1065 					break;
1066 				}
1067 				if (func->type == ZEND_USER_FUNCTION
1068 				 && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
1069 					jit_extension =
1070 						(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array);
1071 					if (UNEXPECTED(!jit_extension)
1072 					 || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)
1073 					 || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) {
1074 						stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
1075 						break;
1076 					}
1077 					func = (zend_function*)jit_extension->op_array;
1078 				}
1079 
1080 #ifndef HAVE_GCC_GLOBAL_REGS
1081 				opline = EX(opline);
1082 #endif
1083 
1084 				if (JIT_G(max_polymorphic_calls) == 0
1085 				 && zend_jit_may_be_polymorphic_call(opline - 1)) {
1086 					func = NULL;
1087 				} else if ((is_megamorphic == ZEND_JIT_EXIT_METHOD_CALL
1088 						 || is_megamorphic == ZEND_JIT_EXIT_CLOSURE_CALL)
1089 						&& trace_buffer[1].opline == opline - 1) {
1090 					func = NULL;
1091 				}
1092 				if (!func) {
1093 					ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC);
1094 				}
1095 				TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func);
1096 			}
1097 			prev_call = EX(call);
1098 		}
1099 
1100 #ifndef HAVE_GCC_GLOBAL_REGS
1101 		opline = EX(opline);
1102 #endif
1103 
1104 		if (UNEXPECTED(opline->opcode == ZEND_HANDLE_EXCEPTION)) {
1105 			/* Abort trace because of exception */
1106 			stop = ZEND_JIT_TRACE_STOP_EXCEPTION;
1107 			break;
1108 		}
1109 
1110 		trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
1111 		if (trace_flags) {
1112 			if (trace_flags & ZEND_JIT_TRACE_JITED) {
1113 				if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
1114 					if ((start & ZEND_JIT_TRACE_START_LOOP) != 0
1115 					 && level + ret_level == 0
1116 					 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) - 1) !=
1117 							ZEND_JIT_TRACE_STOP_COMPILED_LOOP) {
1118 						/* Fail to try close outer loop through side exit.
1119 						   If this doesn't work just link. */
1120 						stop = ZEND_JIT_TRACE_STOP_COMPILED_LOOP;
1121 						break;
1122 					} else {
1123 						stop = ZEND_JIT_TRACE_STOP_LINK;
1124 						break;
1125 					}
1126 				} else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
1127 					if (start != ZEND_JIT_TRACE_START_RETURN) {
1128 						// TODO: We may try to inline function ???
1129 						stop = ZEND_JIT_TRACE_STOP_LINK;
1130 						break;
1131 					}
1132 					if (backtrack_link_to_enter < 0) {
1133 						backtrack_link_to_enter = idx;
1134 						link_to_enter_opline = opline;
1135 					}
1136 				} else {
1137 					stop = ZEND_JIT_TRACE_STOP_LINK;
1138 					break;
1139 				}
1140 			} else if (trace_flags & ZEND_JIT_TRACE_BLACKLISTED) {
1141 				stop = ZEND_JIT_TRACE_STOP_BLACK_LIST;
1142 				break;
1143 			} else if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
1144 				uint8_t bad_stop;
1145 
1146 				if (start != ZEND_JIT_TRACE_START_SIDE) {
1147 					if (opline == orig_opline && level + ret_level == 0) {
1148 						stop = ZEND_JIT_TRACE_STOP_LOOP;
1149 						break;
1150 					}
1151 				}
1152 
1153 				if (start != ZEND_JIT_TRACE_START_SIDE
1154 				 || level + ret_level != 0) {
1155 					/* First try creating a trace for inner loop.
1156 					   If this doesn't work try loop unroling. */
1157 					bad_stop = zend_jit_trace_bad_stop_event(opline,
1158 						JIT_G(blacklist_root_trace) / 2);
1159 					if (bad_stop != ZEND_JIT_TRACE_STOP_INNER_LOOP
1160 					 && bad_stop != ZEND_JIT_TRACE_STOP_LOOP_EXIT) {
1161 						if (start == ZEND_JIT_TRACE_START_SIDE
1162 						 || zend_jit_trace_bad_stop_event(orig_opline,
1163 								JIT_G(blacklist_root_trace) / 2) != ZEND_JIT_TRACE_STOP_INNER_LOOP) {
1164 							stop = ZEND_JIT_TRACE_STOP_INNER_LOOP;
1165 							break;
1166 						}
1167 					}
1168 				}
1169 
1170 				if (opline == last_loop_opline
1171 				 && level == last_loop_level) {
1172 					idx = zend_jit_trace_subtrace(trace_buffer,
1173 						last_loop, idx, ZEND_JIT_TRACE_START_LOOP, op_array, opline);
1174 					start = ZEND_JIT_TRACE_START_LOOP;
1175 					stop = ZEND_JIT_TRACE_STOP_LOOP;
1176 					ret_level = 0;
1177 					break;
1178 				} else if (loop_unroll_limit < JIT_G(max_loop_unrolls)) {
1179 					last_loop = idx;
1180 					last_loop_opline = opline;
1181 					last_loop_level = level;
1182 					loop_unroll_limit++;
1183 				} else {
1184 					stop = ZEND_JIT_TRACE_STOP_LOOP_UNROLL;
1185 					break;
1186 				}
1187 			} else if (trace_flags & ZEND_JIT_TRACE_UNSUPPORTED) {
1188 				TRACE_RECORD(ZEND_JIT_TRACE_VM, 0, opline);
1189 				stop = ZEND_JIT_TRACE_STOP_NOT_SUPPORTED;
1190 				break;
1191 			}
1192 		}
1193 	}
1194 
1195 	end_opline = opline;
1196 	if (!ZEND_JIT_TRACE_STOP_OK(stop)) {
1197 		if (backtrack_recursion > 0) {
1198 			idx = backtrack_recursion;
1199 			stop = ZEND_JIT_TRACE_STOP_RECURSIVE_CALL;
1200 			end_opline = orig_opline;
1201 		} else if (backtrack_ret_recursion > 0) {
1202 			idx = backtrack_ret_recursion;
1203 			ret_level = backtrack_ret_recursion_level;
1204 			stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
1205 			end_opline = orig_opline;
1206 		} else if (backtrack_link_to_enter > 0) {
1207 			if (stop == ZEND_JIT_TRACE_STOP_DEEP_RECURSION
1208 			 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) / 2) ==
1209 					ZEND_JIT_TRACE_STOP_DEEP_RECURSION) {
1210 				idx = backtrack_link_to_enter;
1211 				stop = ZEND_JIT_TRACE_STOP_LINK;
1212 				end_opline = link_to_enter_opline;
1213 			}
1214 		}
1215 	}
1216 
1217 	if (stop == ZEND_JIT_TRACE_STOP_LINK) {
1218 		/* Shrink fake INIT_CALLs */
1219 		while (trace_buffer[idx-1].op == ZEND_JIT_TRACE_INIT_CALL
1220 				&& (trace_buffer[idx-1].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1221 			idx--;
1222 		}
1223 	}
1224 
1225 	TRACE_END(ZEND_JIT_TRACE_END, stop, end_opline);
1226 
1227 #ifdef HAVE_GCC_GLOBAL_REGS
1228 	if (!halt) {
1229 		EX(opline) = opline;
1230 	}
1231 #endif
1232 
1233 #ifdef HAVE_GCC_GLOBAL_REGS
1234 	execute_data = save_execute_data;
1235 	opline = save_opline;
1236 #endif
1237 
1238 	return stop | halt;
1239 }
1240