1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(bool reattached)49 static void zend_jit_trace_startup(bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT root traces buffer!");
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT exit groups buffer!");
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 ZCSG(jit_counters_stopped) = false;
67 } else {
68 zend_jit_traces = ZCSG(jit_traces);
69 if (!zend_jit_traces) {
70 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not obtain JIT traces buffer!");
71 }
72 zend_jit_exit_groups = ZCSG(jit_exit_groups);
73 if (!zend_jit_exit_groups) {
74 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not obtain JIT exit groups buffer!");
75 }
76 }
77
78 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
79 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
80
81 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
82 if (JIT_G(exit_counters) == NULL) {
83 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT exit counters buffer!");
84 }
85 }
86
zend_jit_trace_allocate_exit_point(uint32_t n)87 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
88 {
89 const void *group = NULL;
90
91 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
92 return NULL;
93 }
94 do {
95 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
96 if (!group) {
97 return NULL;
98 }
99 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
100 group;
101 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
102 } while (n >= ZEND_JIT_EXIT_NUM);
103 return (const void*)
104 ((const char*)group +
105 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
106 }
107
zend_jit_trace_get_exit_addr(uint32_t n)108 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
109 {
110 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
111 return zend_jit_trace_allocate_exit_point(n);
112 }
113 return (const void*)
114 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
115 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
116 }
117
zend_jit_exit_point_by_addr(const void * addr)118 static uint32_t zend_jit_exit_point_by_addr(const void *addr)
119 {
120 uint32_t n = (ZEND_JIT_EXIT_NUM + (ZEND_JIT_EXIT_POINTS_PER_GROUP - 1)) / ZEND_JIT_EXIT_POINTS_PER_GROUP;
121 uint32_t i;
122
123 for (i = 0; i < n; i++) {
124 if ((char*)addr >= (char*)zend_jit_exit_groups[i]
125 && (char*)addr <= (char*)zend_jit_exit_groups[i] + ((ZEND_JIT_EXIT_POINTS_PER_GROUP - 1) * ZEND_JIT_EXIT_POINTS_SPACING)) {
126 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
127 (((char*)addr - (char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING);
128 }
129 }
130 return (uint32_t)-1;
131 }
132
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)133 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
134 {
135 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
136 uint32_t exit_point;
137 const zend_op_array *op_array;
138 uint32_t stack_offset = (uint32_t)-1;
139 uint32_t stack_size;
140 zend_jit_trace_stack *stack = NULL;
141
142 if (delayed_call_chain) {
143 assert(to_opline != NULL); /* CALL and IP share the same register */
144 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
145 }
146 if (JIT_G(current_frame)) {
147 op_array = &JIT_G(current_frame)->func->op_array;
148 stack_size = op_array->last_var + op_array->T;
149 if (stack_size) {
150 stack = JIT_G(current_frame)->stack;
151 do {
152 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
153 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
154 || STACK_REF(stack, stack_size-1) != IR_UNUSED
155 ) {
156 break;
157 }
158 stack_size--;
159 } while (stack_size);
160 }
161 } else {
162 op_array = NULL;
163 stack_size = 0;
164 }
165
166 /* Try to reuse exit points */
167 if (to_opline != NULL
168 && !(flags & ZEND_JIT_EXIT_METHOD_CALL)
169 && t->exit_count > 0) {
170 uint32_t i = t->exit_count;
171
172 do {
173 i--;
174 if (stack_size == 0
175 || (t->exit_info[i].stack_size >= stack_size
176 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
177 if (t->exit_info[i].opline == to_opline
178 && t->exit_info[i].flags == flags
179 && t->exit_info[i].stack_size == stack_size) {
180 return i;
181 }
182 }
183 } while (i > 0);
184 }
185
186 exit_point = t->exit_count;
187 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
188 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
189 stack_offset = t->stack_map_size;
190 t->stack_map_size += stack_size;
191 // TODO: reduce number of reallocations ???
192 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
193 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
194 }
195 t->exit_count++;
196 t->exit_info[exit_point].opline = to_opline;
197 t->exit_info[exit_point].op_array = op_array;
198 t->exit_info[exit_point].flags = flags;
199 t->exit_info[exit_point].stack_size = stack_size;
200 t->exit_info[exit_point].stack_offset = stack_offset;
201 t->exit_info[exit_point].poly_func_ref = 0;
202 t->exit_info[exit_point].poly_this_ref = 0;
203 t->exit_info[exit_point].poly_func_reg = ZREG_NONE;
204 t->exit_info[exit_point].poly_this_reg = ZREG_NONE;
205 }
206
207 return exit_point;
208 }
209
zend_jit_trace_add_code(const void * start,uint32_t size)210 static void zend_jit_trace_add_code(const void *start, uint32_t size)
211 {
212 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
213
214 t->code_start = start;
215 t->code_size = size;
216 }
217
218 /**
219 * Locate a trace in the #zend_jit_traces array with the specified
220 * #code_start address.
221 *
222 * @return the #zend_jit_traces index or 0 if no such #code_start
223 * address was found
224 */
zend_jit_find_trace(const void * addr)225 static uint32_t zend_jit_find_trace(const void *addr)
226 {
227 uint32_t i;
228
229 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
230 if (zend_jit_traces[i].code_start == addr) {
231 return i;
232 }
233 }
234 return 0;
235 }
236
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)237 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
238 {
239 smart_str buf = {0};
240
241 smart_str_appends(&buf, TRACE_PREFIX);
242 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
243 smart_str_appendc(&buf, '$');
244 if (op_array->function_name) {
245 if (op_array->scope) {
246 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
247 smart_str_appends(&buf, "::");
248 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
249 } else {
250 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
251 }
252 } else if (op_array->filename) {
253 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
254 }
255 smart_str_appendc(&buf, '$');
256 smart_str_append_long(&buf, (zend_long)lineno);
257 smart_str_0(&buf);
258 return buf.s;
259 }
260
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)261 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
262 {
263 switch (opline->opcode) {
264 case ZEND_IS_IDENTICAL:
265 case ZEND_IS_NOT_IDENTICAL:
266 case ZEND_IS_EQUAL:
267 case ZEND_IS_NOT_EQUAL:
268 case ZEND_IS_SMALLER:
269 case ZEND_IS_SMALLER_OR_EQUAL:
270 case ZEND_CASE:
271 case ZEND_CASE_STRICT:
272 case ZEND_ISSET_ISEMPTY_CV:
273 case ZEND_ISSET_ISEMPTY_VAR:
274 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
275 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
276 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
277 case ZEND_INSTANCEOF:
278 case ZEND_TYPE_CHECK:
279 case ZEND_DEFINED:
280 case ZEND_IN_ARRAY:
281 case ZEND_ARRAY_KEY_EXISTS:
282 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
283 /* smart branch */
284 return 1;
285 }
286 break;
287 case ZEND_JMPZ:
288 case ZEND_JMPNZ:
289 case ZEND_JMPZ_EX:
290 case ZEND_JMPNZ_EX:
291 case ZEND_JMP_SET:
292 case ZEND_COALESCE:
293 case ZEND_JMP_NULL:
294 case ZEND_FE_RESET_R:
295 case ZEND_FE_RESET_RW:
296 case ZEND_ASSERT_CHECK:
297 case ZEND_FE_FETCH_R:
298 case ZEND_FE_FETCH_RW:
299 case ZEND_SWITCH_LONG:
300 case ZEND_SWITCH_STRING:
301 case ZEND_MATCH:
302 case ZEND_BIND_INIT_STATIC_OR_JMP:
303 case ZEND_JMP_FRAMELESS:
304 /* branch opcodes */
305 return 1;
306 case ZEND_NEW:
307 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
308 /* NEW may skip constructor without arguments */
309 return 1;
310 }
311 break;
312 case ZEND_CATCH:
313 case ZEND_FAST_CALL:
314 case ZEND_FAST_RET:
315 case ZEND_GENERATOR_CREATE:
316 case ZEND_GENERATOR_RETURN:
317 case ZEND_YIELD:
318 case ZEND_YIELD_FROM:
319 case ZEND_INCLUDE_OR_EVAL:
320 case ZEND_MATCH_ERROR:
321 /* unsupported */
322 return 1;
323 case ZEND_DO_FCALL:
324 /* potentially polymorphic call */
325 return 1;
326 #if 0
327 case ZEND_DO_UCALL:
328 case ZEND_DO_FCALL_BY_NAME:
329 /* monomorphic call */
330 // TODO: recompilation may change target ???
331 return 0;
332 #endif
333 case ZEND_RETURN_BY_REF:
334 case ZEND_RETURN:
335 /* return */
336 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
337 default:
338 break;
339 }
340 return 0;
341 }
342
zend_jit_trace_type_to_info_ex(uint8_t type,uint32_t info)343 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(uint8_t type, uint32_t info)
344 {
345 if (type == IS_UNKNOWN) {
346 return info;
347 }
348 ZEND_ASSERT(info & (1 << type));
349 if (type < IS_STRING) {
350 return (1 << type);
351 } else if (type != IS_ARRAY) {
352 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
353 } else {
354 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
355 }
356 }
357
zend_jit_trace_type_to_info(uint8_t type)358 static zend_always_inline uint32_t zend_jit_trace_type_to_info(uint8_t type)
359 {
360 return zend_jit_trace_type_to_info_ex(type, -1);
361 }
362
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)363 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
364 {
365 if (var >= op_array->last_var) {
366 return NO_ALIAS;
367 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
368 return SYMTABLE_ALIAS;
369 } else if (ssa->vars) {
370 return ssa->vars[var].alias;
371 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
372 return HTTP_RESPONSE_HEADER_ALIAS;
373 }
374 return NO_ALIAS;
375 }
376
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)377 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
378 int ssa_var,
379 uint8_t op_type)
380 {
381 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
382
383 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
384 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
385 info->type |= MAY_BE_GUARD;
386 } else {
387 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
388 }
389 }
390 }
391
392 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
393 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
394 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
395 } \
396 } while (0)
397
398 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
399 if (op_type != IS_UNKNOWN) { \
400 if ((op_info & MAY_BE_GUARD) != 0) { \
401 if (!zend_jit_type_guard(&ctx, opline, _var, op_type)) { \
402 goto jit_failure; \
403 } \
404 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
405 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
406 op_info = zend_jit_trace_type_to_info(op_type); \
407 } else { \
408 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
409 op_info &= ~MAY_BE_GUARD; \
410 ssa->var_info[_ssa_var].type &= op_info; \
411 } \
412 } \
413 } \
414 } while (0)
415
416 #define ADD_OP1_TRACE_GUARD() \
417 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
418 #define ADD_OP2_TRACE_GUARD() \
419 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
420 #define ADD_OP1_DATA_TRACE_GUARD() \
421 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
422
423 #define CHECK_OP1_TRACE_TYPE() \
424 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
425 #define CHECK_OP2_TRACE_TYPE() \
426 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
427 #define CHECK_OP1_DATA_TRACE_TYPE() \
428 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
429
zend_jit_trace_frame_size(const zend_op_array * op_array,uint32_t num_args)430 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array, uint32_t num_args)
431 {
432 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
433 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
434 } else if (op_array) {
435 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
436 } else {
437 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(num_args * sizeof(zend_jit_trace_stack)));
438 }
439 }
440
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array,uint32_t num_args)441 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array, uint32_t num_args)
442 {
443 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array, num_args));
444 }
445
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)446 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
447 {
448 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array, 0));
449 }
450
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,uint8_t type)451 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type)
452 {
453 zend_jit_trace_stack *stack = call->stack;
454 const zend_op_array *op_array = &call->func->op_array;
455 uint32_t arg_num = opline->op2.num;
456
457 if (arg_num > op_array->num_args) {
458 return;
459 }
460 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
461 zend_arg_info *arg_info;
462
463 ZEND_ASSERT(arg_num <= op_array->num_args);
464 arg_info = &op_array->arg_info[arg_num-1];
465
466 if (ZEND_TYPE_IS_SET(arg_info->type)) {
467 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
468 return;
469 }
470 }
471 }
472 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
473 }
474
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)475 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
476 {
477 if (func
478 && func->type == ZEND_INTERNAL_FUNCTION
479 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
480 && arg_num < func->internal_function.num_args) {
481 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
482
483 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
484 && ZEND_TYPE_IS_SET(arg_info->type)
485 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
486 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
487 && JIT_G(current_frame)
488 && JIT_G(current_frame)->call
489 && JIT_G(current_frame)->call->func) {
490 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
491
492 if (type != IS_UNKNOWN
493 && type < IS_STRING
494 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
495 return 0;
496 }
497 }
498 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
499 const zend_op *opline = call_info->arg_info[arg_num].opline;
500
501 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
502 zval *zv = RT_CONSTANT(opline, opline->op1);
503
504 if (!Z_REFCOUNTED_P(zv)) {
505 uint32_t type = Z_TYPE_P(zv);
506
507 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
508 if (type != IS_ARRAY
509 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
510 return 0;
511 }
512 }
513 }
514 }
515 }
516 }
517
518 return 1;
519 }
520
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)521 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
522 {
523 zend_jit_op_array_trace_extension *jit_extension;
524 zend_ssa *ssa;
525
526 jit_extension =
527 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
528 jit_extension->func_info.num = 0;
529 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
530 | ZEND_FUNC_JIT_ON_PROF_REQUEST
531 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
532 | ZEND_FUNC_JIT_ON_HOT_TRACE;
533 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
534 ssa = &jit_extension->func_info.ssa;
535
536 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
537 do {
538 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
539 break;
540 }
541
542 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
543 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info);
544 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
545 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
546 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
547 }
548 }
549
550 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
551 break;
552 }
553
554 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
555 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
556 }
557 return ssa;
558 } while (0);
559 }
560
561 memset(ssa, 0, sizeof(zend_ssa));
562 ssa->cfg.blocks_count = 1;
563
564 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
565 zend_cfg cfg;
566 void *checkpoint = zend_arena_checkpoint(CG(arena));
567
568 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
569 ssa->cfg.flags = cfg.flags;
570 } else{
571 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
572 }
573
574 /* TODO: move this to zend_cfg.c ? */
575 if (!op_array->function_name) {
576 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
577 }
578
579 zend_arena_release(&CG(arena), checkpoint);
580 }
581
582 return ssa;
583 }
584
585 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
586 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
587
zend_jit_trace_op_len(const zend_op * opline)588 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
589 {
590 int len;
591
592 switch (opline->opcode) {
593 case ZEND_ASSIGN_DIM:
594 case ZEND_ASSIGN_OBJ:
595 case ZEND_ASSIGN_STATIC_PROP:
596 case ZEND_ASSIGN_DIM_OP:
597 case ZEND_ASSIGN_OBJ_OP:
598 case ZEND_ASSIGN_STATIC_PROP_OP:
599 case ZEND_ASSIGN_OBJ_REF:
600 case ZEND_ASSIGN_STATIC_PROP_REF:
601 case ZEND_FRAMELESS_ICALL_3:
602 return 2; /* OP_DATA */
603 case ZEND_RECV_INIT:
604 len = 1;
605 opline++;
606 while (opline->opcode == ZEND_RECV_INIT) {
607 len++;
608 opline++;
609 }
610 return len;
611 case ZEND_BIND_GLOBAL:
612 len = 1;
613 opline++;
614 while (opline->opcode == ZEND_BIND_GLOBAL) {
615 len++;
616 opline++;
617 }
618 return len;
619 // case ZEND_IS_IDENTICAL:
620 // case ZEND_IS_NOT_IDENTICAL:
621 // case ZEND_IS_EQUAL:
622 // case ZEND_IS_NOT_EQUAL:
623 // case ZEND_IS_SMALLER:
624 // case ZEND_IS_SMALLER_OR_EQUAL:
625 // case ZEND_CASE:
626 // case ZEND_ISSET_ISEMPTY_CV:
627 // case ZEND_ISSET_ISEMPTY_VAR:
628 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
629 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
630 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
631 // case ZEND_INSTANCEOF:
632 // case ZEND_TYPE_CHECK:
633 // case ZEND_DEFINED:
634 // case ZEND_IN_ARRAY:
635 // case ZEND_ARRAY_KEY_EXISTS:
636 default:
637 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
638 return 2; /* JMPZ/JMPNZ */
639 }
640 return 1;
641 }
642 }
643
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)644 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
645 {
646 const zend_op_array *op_array;
647 zend_jit_trace_rec *p;
648 int k, vars_count;
649 zend_bitset use, def;
650 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
651 uint32_t set_size;
652 zend_ssa_phi *prev = NULL;
653 int level = 0;
654 ALLOCA_FLAG(use_heap);
655
656 op_array = trace_buffer->op_array;
657 set_size = zend_bitset_len(op_array->last_var + op_array->T);
658 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
659 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
660 def = use + set_size;
661 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
662 for (;;p++) {
663 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
664 const zend_op *opline = p->opline;
665 int len;
666
667 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
668 len = zend_jit_trace_op_len(opline);
669 while (len > 1) {
670 opline++;
671 if (opline->opcode != ZEND_OP_DATA) {
672 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
673 }
674 len--;
675 }
676 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
677 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
678 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
679 level++;
680 } else if (p->op == ZEND_JIT_TRACE_BACK) {
681 if (level == 0) {
682 // Phi for recursive calls and returns are not supported yet ???
683 assert(0);
684 } else {
685 level--;
686 }
687 } else if (p->op == ZEND_JIT_TRACE_END) {
688 break;
689 }
690 }
691
692 zend_bitset_intersection(use, def, set_size);
693
694 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
695 vars_count = op_array->last_var;
696 } else {
697 vars_count = op_array->last_var + op_array->T;
698 }
699 for (k = 0; k < vars_count; k++) {
700 if (zend_bitset_in(use, k)) {
701 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
702 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
703 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
704 sizeof(void*) * 2);
705 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
706 phi->sources[0] = STACK_VAR(stack, k);
707 phi->sources[1] = -1;
708 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
709 phi->pi = -1;
710 phi->var = k;
711 phi->ssa_var = ssa_vars_count;
712 SET_STACK_VAR(stack, k, ssa_vars_count);
713 ssa_vars_count++;
714 phi->block = 1;
715 if (prev) {
716 prev->next = phi;
717 } else {
718 tssa->blocks[1].phis = phi;
719 }
720 prev = phi;
721 }
722 }
723
724 free_alloca(use, use_heap);
725
726 return ssa_vars_count;
727 }
728
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)729 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
730 {
731 zend_ssa_phi *prev = NULL;
732 const zend_op_array *op_array = trace_buffer->op_array;
733 const zend_op *opline = trace_buffer[1].opline;
734 int count = opline - op_array->opcodes;
735 int i;
736
737 for(i = 0; i < count; i++) {
738 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
739 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
740 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
741 sizeof(void*) * 2);
742 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
743 phi->sources[0] = STACK_VAR(stack, i);
744 phi->sources[1] = -1;
745 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
746 phi->pi = -1;
747 phi->var = i;
748 phi->ssa_var = ssa_vars_count;
749 SET_STACK_VAR(stack, i, ssa_vars_count);
750 ssa_vars_count++;
751 phi->block = 1;
752 if (prev) {
753 prev->next = phi;
754 } else {
755 tssa->blocks[1].phis = phi;
756 }
757 prev = phi;
758 }
759 return ssa_vars_count;
760 }
761
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)762 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
763 {
764 const zend_op *opline = trace_buffer[1].opline - 1;
765 int i;
766
767 if (RETURN_VALUE_USED(opline)) {
768 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
769 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
770 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
771 sizeof(void*) * 2);
772
773 i = EX_VAR_TO_NUM(opline->result.var);
774 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
775 phi->sources[0] = STACK_VAR(stack, i);
776 phi->sources[1] = -1;
777 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
778 phi->pi = -1;
779 phi->var = i;
780 phi->ssa_var = ssa_vars_count;
781 SET_STACK_VAR(stack, i, ssa_vars_count);
782 ssa_vars_count++;
783 phi->block = 1;
784 tssa->blocks[1].phis = phi;
785 }
786 return ssa_vars_count;
787 }
788
zend_jit_trace_is_false_loop(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa)789 static bool zend_jit_trace_is_false_loop(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa)
790 {
791 const zend_op *opline;
792 uint32_t b;
793 zend_basic_block *bb;
794
795 ZEND_ASSERT(tssa->cfg.blocks_count == 2);
796 ZEND_ASSERT(tssa->cfg.blocks[1].len > 0);
797
798 b = ssa->cfg.map[tssa_opcodes[0] - op_array->opcodes];
799 opline = tssa_opcodes[tssa->cfg.blocks[1].len - 1];
800 if (opline >= op_array->opcodes && opline < op_array->opcodes + op_array->last) {
801 bb = ssa->cfg.blocks + ssa->cfg.map[opline - op_array->opcodes];
802 return bb->loop_header != b;
803 } else {
804 return 0;
805 }
806 }
807
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var,const zend_op * opline)808 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array,
809 const zend_ssa *ssa,
810 const zend_op **tssa_opcodes,
811 zend_ssa *tssa,
812 int ssa_var,
813 const zend_op *opline)
814 {
815 int var, use, def, src;
816 zend_ssa_op *op;
817
818 if (tssa->vars[ssa_var].definition_phi) {
819 uint32_t b = ssa->cfg.map[tssa_opcodes[0] - op_array->opcodes];
820 zend_basic_block *bb = ssa->cfg.blocks + b;
821
822 if ((bb->flags & ZEND_BB_LOOP_HEADER)
823 && !zend_jit_trace_is_false_loop(op_array, ssa, tssa_opcodes, tssa)) {
824 zend_ssa_phi *phi = ssa->blocks[b].phis;
825 zend_ssa_phi *pi = NULL;
826
827 var = tssa->vars[ssa_var].var;
828 while (phi) {
829 if (ssa->vars[phi->ssa_var].var == var) {
830 if (phi->pi >= 0) {
831 pi = phi;
832 } else {
833 src = phi->ssa_var;
834 goto copy_info;
835 }
836 }
837 phi = phi->next;
838 }
839 if (pi) {
840 src = pi->ssa_var;
841 goto copy_info;
842 }
843 #if 0
844 while (bb->idom >= 0) {
845 uint32_t n;
846
847 b = bb->idom;
848 bb = ssa->cfg.blocks + b;
849
850 for (n = bb->len, op = ssa->ops + bb->start + n; n > 0; n--) {
851 op--;
852 if (op->result_def >= 0 && ssa->vars[op->result_def].var == var) {
853 src = op->result_def;
854 goto copy_info;
855 } else if (op->op2_def >= 0 && ssa->vars[op->op2_def].var == var) {
856 src = op->op2_def;
857 goto copy_info;
858 } else if (op->op1_def >= 0 && ssa->vars[op->op1_def].var == var) {
859 src = op->op1_def;
860 goto copy_info;
861 }
862 }
863
864 phi = ssa->blocks[b].phis;
865 zend_ssa_phi *pi = NULL;
866 while (phi) {
867 if (ssa->vars[phi->ssa_var].var == var) {
868 if (phi->pi >= 0) {
869 pi = phi;
870 } else {
871 src = phi->ssa_var;
872 goto copy_info;
873 }
874 }
875 phi = phi->next;
876 }
877 if (pi) {
878 src = pi->ssa_var;
879 goto copy_info;
880 }
881 }
882 #endif
883 }
884 } else if (tssa->vars[ssa_var].definition >= 0) {
885 def = tssa->vars[ssa_var].definition;
886 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
887 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
888 if (tssa->ops[def].op1_def == ssa_var) {
889 src = op->op1_def;
890 } else if (tssa->ops[def].op2_def == ssa_var) {
891 src = op->op2_def;
892 } else if (tssa->ops[def].result_def == ssa_var) {
893 src = op->result_def;
894 } else {
895 assert(0);
896 return 0;
897 }
898 goto copy_info;
899 }
900
901 if (tssa->vars[ssa_var].phi_use_chain) {
902 // TODO: this may be incorrect ???
903 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
904 } else {
905 var = ssa_var;
906 }
907 use = tssa->vars[var].use_chain;
908 if (use >= 0) {
909 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
910 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
911 if (tssa->ops[use].op1_use == var) {
912 src = op->op1_use;
913 } else if (tssa->ops[use].op2_use == var) {
914 src = op->op2_use;
915 } else if (tssa->ops[use].result_use == var) {
916 src = op->result_use;
917 } else {
918 assert(0);
919 return 0;
920 }
921 if (opline) {
922 /* Try to find a difinition in SSA dominators tree */
923 var = tssa->vars[ssa_var].var;
924 uint32_t op_num = opline - op_array->opcodes;
925 uint32_t b = ssa->cfg.map[op_num];
926 zend_basic_block *bb = ssa->cfg.blocks + b;
927 zend_ssa_phi *pi, *phi;
928
929 while (1) {
930 while (op_num > bb->start) {
931 op_num--;
932 op = ssa->ops + op_num;
933 if (op->result_def >= 0 && ssa->vars[op->result_def].var == var) {
934 src = op->result_def;
935 goto copy_info;
936 } else if (op->op2_def >= 0 && ssa->vars[op->op2_def].var == var) {
937 src = op->op2_def;
938 goto copy_info;
939 } else if (op->op1_def >= 0 && ssa->vars[op->op1_def].var == var) {
940 src = op->op1_def;
941 goto copy_info;
942 }
943 }
944 phi = ssa->blocks[b].phis;
945 pi = NULL;
946 while (phi) {
947 if (ssa->vars[phi->ssa_var].var == var) {
948 if (phi->pi >= 0) {
949 pi = phi;
950 } else {
951 src = phi->ssa_var;
952 goto copy_info;
953 }
954 }
955 phi = phi->next;
956 }
957 if (pi) {
958 src = pi->ssa_var;
959 goto copy_info;
960 }
961 if (bb->idom < 0) {
962 break;
963 }
964 b = bb->idom;
965 bb = ssa->cfg.blocks + b;
966 op_num = bb->start + bb->len;
967 }
968 }
969 goto copy_info;
970 }
971 return 0;
972
973 copy_info:
974 tssa->vars[ssa_var].no_val = ssa->vars[src].no_val;
975 tssa->vars[ssa_var].alias = ssa->vars[src].alias;
976 memcpy(&tssa->var_info[ssa_var], &ssa->var_info[src], sizeof(zend_ssa_var_info));
977 return 1;
978 }
979
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)980 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
981 {
982 zend_ssa_range tmp;
983 int def = tssa->vars[ssa_var].definition;
984
985 if (tssa->vars[ssa_var].alias == NO_ALIAS
986 && zend_inference_propagate_range(op_array, tssa, tssa_opcodes[def], &tssa->ops[def], ssa_var, &tmp)) {
987 tssa->var_info[ssa_var].range.min = tmp.min;
988 tssa->var_info[ssa_var].range.max = tmp.max;
989 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
990 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
991 tssa->var_info[ssa_var].has_range = 1;
992 }
993 }
994
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)995 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
996 {
997 int def;
998 zend_ssa_op *op;
999 zend_ssa_var_info *info;
1000 unsigned int no_val;
1001 zend_ssa_alias_kind alias;
1002
1003 def = tssa->vars[ssa_var].definition;
1004 if (def >= 0) {
1005 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
1006 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
1007 if (tssa->ops[def].op1_def == ssa_var) {
1008 no_val = ssa->vars[op->op1_def].no_val;
1009 alias = ssa->vars[op->op1_def].alias;
1010 info = ssa->var_info + op->op1_def;
1011 } else if (tssa->ops[def].op2_def == ssa_var) {
1012 no_val = ssa->vars[op->op2_def].no_val;
1013 alias = ssa->vars[op->op2_def].alias;
1014 info = ssa->var_info + op->op2_def;
1015 } else if (tssa->ops[def].result_def == ssa_var) {
1016 no_val = ssa->vars[op->result_def].no_val;
1017 alias = ssa->vars[op->result_def].alias;
1018 info = ssa->var_info + op->result_def;
1019 } else {
1020 assert(0);
1021 return;
1022 }
1023
1024 tssa->vars[ssa_var].no_val = no_val;
1025 tssa->vars[ssa_var].alias = alias;
1026
1027 if (!(info->type & MAY_BE_REF)) {
1028 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
1029 }
1030
1031 if (info->has_range) {
1032 if (tssa->var_info[ssa_var].has_range) {
1033 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
1034 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
1035 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
1036 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
1037 } else {
1038 tssa->var_info[ssa_var].has_range = 1;
1039 tssa->var_info[ssa_var].range = info->range;
1040 }
1041 }
1042 }
1043 }
1044
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)1045 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
1046 {
1047 int def;
1048 zend_ssa_op *op;
1049 zend_ssa_var_info *info;
1050
1051 def = tssa->vars[ssa_var].definition;
1052 if (def >= 0) {
1053 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
1054 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
1055 if (tssa->ops[def].op1_def == ssa_var) {
1056 info = ssa->var_info + op->op1_def;
1057 } else if (tssa->ops[def].op2_def == ssa_var) {
1058 info = ssa->var_info + op->op2_def;
1059 } else if (tssa->ops[def].result_def == ssa_var) {
1060 info = ssa->var_info + op->result_def;
1061 } else {
1062 assert(0);
1063 return 0;
1064 }
1065 tssa->var_info[ssa_var].type &= info->type;
1066 if (info->ce) {
1067 if (tssa->var_info[ssa_var].ce) {
1068 if (tssa->var_info[ssa_var].ce != info->ce) {
1069 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
1070 /* everything fine */
1071 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
1072 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
1073 } else {
1074 // TODO: classes may implement the same interface ???
1075 //ZEND_UNREACHABLE();
1076 }
1077 }
1078 tssa->var_info[ssa_var].is_instanceof =
1079 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
1080 } else {
1081 tssa->var_info[ssa_var].ce = info->ce;
1082 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
1083 }
1084 }
1085 if (info->has_range) {
1086 if (tssa->var_info[ssa_var].has_range) {
1087 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
1088 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
1089 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
1090 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
1091 } else {
1092 tssa->var_info[ssa_var].has_range = 1;
1093 tssa->var_info[ssa_var].range = info->range;
1094 }
1095 }
1096 return 1;
1097 }
1098 return 0;
1099 }
1100
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)1101 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
1102 {
1103 while (1) {
1104 if (p->op == ZEND_JIT_TRACE_VM) {
1105 if (p->opline->opcode == ZEND_DO_UCALL
1106 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
1107 || p->opline->opcode == ZEND_DO_FCALL) {
1108 if (p->opline->result_type != IS_UNUSED) {
1109 return ssa_op->result_def;
1110 }
1111 }
1112 return -1;
1113 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1114 /*skip */
1115 } else {
1116 return -1;
1117 }
1118 p--;
1119 }
1120 }
1121
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1122 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1123 {
1124 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1125 p--;
1126 while (1) {
1127 if (p->op == ZEND_JIT_TRACE_VM) {
1128 if (p->opline->opcode == ZEND_INIT_FCALL
1129 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1130 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1131 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1132 || p->opline->opcode == ZEND_INIT_USER_CALL
1133 || p->opline->opcode == ZEND_NEW
1134 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1135 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
1136 || p->opline->opcode == ZEND_INIT_PARENT_PROPERTY_HOOK_CALL) {
1137 return p->opline;
1138 }
1139 return NULL;
1140 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1141 /*skip */
1142 } else {
1143 return NULL;
1144 }
1145 p--;
1146 }
1147 } else {
1148 const zend_op *opline = NULL;
1149 int call_level = 0;
1150
1151 p++;
1152 while (1) {
1153 if (p->op == ZEND_JIT_TRACE_VM) {
1154 opline = p->opline;
1155 break;
1156 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1157 call_level++;
1158 /*skip */
1159 } else {
1160 return NULL;
1161 }
1162 p--;
1163 }
1164 if (opline) {
1165 while (opline > op_array->opcodes) {
1166 opline--;
1167 switch (opline->opcode) {
1168 case ZEND_INIT_FCALL:
1169 case ZEND_INIT_FCALL_BY_NAME:
1170 case ZEND_INIT_NS_FCALL_BY_NAME:
1171 case ZEND_INIT_METHOD_CALL:
1172 case ZEND_INIT_DYNAMIC_CALL:
1173 case ZEND_INIT_STATIC_METHOD_CALL:
1174 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
1175 case ZEND_INIT_USER_CALL:
1176 case ZEND_NEW:
1177 if (call_level == 0) {
1178 return opline;
1179 }
1180 call_level--;
1181 break;
1182 case ZEND_DO_FCALL:
1183 case ZEND_DO_ICALL:
1184 case ZEND_DO_UCALL:
1185 case ZEND_DO_FCALL_BY_NAME:
1186 case ZEND_CALLABLE_CONVERT:
1187 call_level++;
1188 break;
1189 }
1190 }
1191 }
1192 }
1193 return NULL;
1194 }
1195
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1196 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1197 {
1198 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1199 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1200 int idx = tssa->vars[var].definition;
1201
1202 if (idx >= 0) {
1203 if (tssa->ops[idx].op1_def == var) {
1204 const zend_op *opline = ssa_opcodes[idx];
1205 if (opline->opcode == ZEND_PRE_DEC
1206 || opline->opcode == ZEND_PRE_INC
1207 || opline->opcode == ZEND_POST_DEC
1208 || opline->opcode == ZEND_POST_INC) {
1209 if (tssa->ops[idx].op1_use >= 0
1210 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1211 return 0;
1212 }
1213 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1214 return 0;
1215 }
1216 return 1;
1217 } else if (opline->opcode == ZEND_ASSIGN_OP
1218 && (opline->extended_value == ZEND_ADD
1219 || opline->extended_value == ZEND_SUB
1220 || opline->extended_value == ZEND_MUL)) {
1221 if ((opline->op2_type & (IS_VAR|IS_CV))
1222 && tssa->ops[idx].op2_use >= 0
1223 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1224 return 0;
1225 }
1226 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1227 return 0;
1228 }
1229 if (opline->op2_type == IS_CONST) {
1230 zval *zv = RT_CONSTANT(opline, opline->op2);
1231 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1232 return 0;
1233 }
1234 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1235 return 0;
1236 }
1237 return 1;
1238 }
1239 }
1240 if (tssa->ops[idx].result_def == var) {
1241 const zend_op *opline = ssa_opcodes[idx];
1242 if (opline->opcode == ZEND_ADD
1243 || opline->opcode == ZEND_SUB
1244 || opline->opcode == ZEND_MUL) {
1245 if ((opline->op1_type & (IS_VAR|IS_CV))
1246 && tssa->ops[idx].op1_use >= 0
1247 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1248 return 0;
1249 }
1250 if ((opline->op2_type & (IS_VAR|IS_CV))
1251 && tssa->ops[idx].op2_use >= 0
1252 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1253 return 0;
1254 }
1255 if (opline->op1_type == IS_CONST) {
1256 zval *zv = RT_CONSTANT(opline, opline->op1);
1257 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1258 return 0;
1259 }
1260 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1261 return 0;
1262 }
1263 if (opline->op2_type == IS_CONST) {
1264 zval *zv = RT_CONSTANT(opline, opline->op2);
1265 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1266 return 0;
1267 }
1268 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1269 return 0;
1270 }
1271 return 1;
1272 } else if (opline->opcode == ZEND_PRE_DEC
1273 || opline->opcode == ZEND_PRE_INC
1274 || opline->opcode == ZEND_POST_DEC
1275 || opline->opcode == ZEND_POST_INC) {
1276 if ((opline->op1_type & (IS_VAR|IS_CV))
1277 && tssa->ops[idx].op1_use >= 0
1278 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1279 return 0;
1280 }
1281 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1282 return 0;
1283 }
1284 return 1;
1285 }
1286 }
1287 }
1288 }
1289 return 0;
1290 }
1291
1292 typedef struct _zend_tssa {
1293 zend_ssa ssa;
1294 const zend_op **tssa_opcodes;
1295 int used_stack;
1296 } zend_tssa;
1297
1298 static const zend_op _nop_opcode = {0};
1299
find_trampoline_num_args(zend_jit_trace_rec * start,zend_jit_trace_rec * p)1300 static uint32_t find_trampoline_num_args(zend_jit_trace_rec *start, zend_jit_trace_rec *p)
1301 {
1302 int inline_level = 0, call_level = 0;
1303
1304 p--;
1305 while (p != start) {
1306 if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1307 if (inline_level == 0) {
1308 if (call_level == 0) {
1309 ZEND_ASSERT(!p->op_array);
1310 return ZEND_JIT_TRACE_NUM_ARGS(p->info);
1311 } else {
1312 call_level--;
1313 }
1314 }
1315 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1316 if (inline_level == 0) {
1317 call_level++;
1318 }
1319 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1320 if (inline_level) {
1321 inline_level--;
1322 } else {
1323 return 0;
1324 }
1325 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1326 inline_level++;
1327 }
1328 p--;
1329 }
1330 return 0;
1331 }
1332
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1333 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1334 {
1335 zend_ssa *tssa;
1336 zend_ssa_op *ssa_ops, *op;
1337 zend_ssa_var *ssa_vars;
1338 zend_ssa_var_info *ssa_var_info;
1339 const zend_op_array *op_array;
1340 const zend_op *opline;
1341 const zend_op **ssa_opcodes;
1342 zend_jit_trace_rec *p;
1343 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1344 zend_jit_trace_stack *stack;
1345 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1346 uint32_t optimization_level = 0;
1347 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1348 size_t frame_size, stack_top, stack_size, stack_bottom;
1349 zend_jit_op_array_trace_extension *jit_extension;
1350 zend_ssa *ssa;
1351 zend_jit_trace_stack_frame *frame, *top, *call;
1352 zend_ssa_var_info return_value_info;
1353
1354 /* 1. Count number of TSSA opcodes;
1355 * Count number of activation frames;
1356 * Calculate size of abstract stack;
1357 * Construct regular SSA for involved op_array */
1358 op_array = trace_buffer->op_array;
1359 stack_top = stack_size = zend_jit_trace_frame_size(op_array, 0);
1360 stack_bottom = 0;
1361 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1362 ssa_ops_count = 0;
1363 call_level = 0;
1364 level = 0;
1365 num_op_arrays = 0;
1366 /* Remember op_array to cleanup */
1367 op_arrays[num_op_arrays++] = op_array;
1368 /* Build SSA */
1369 ssa = zend_jit_trace_build_ssa(op_array, script);
1370 for (;;p++) {
1371 if (p->op == ZEND_JIT_TRACE_VM) {
1372 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1373 const zend_op *opline = p->opline;
1374
1375 switch (opline->opcode) {
1376 case ZEND_INCLUDE_OR_EVAL:
1377 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1378 break;
1379 case ZEND_FETCH_R:
1380 case ZEND_FETCH_W:
1381 case ZEND_FETCH_RW:
1382 case ZEND_FETCH_FUNC_ARG:
1383 case ZEND_FETCH_IS:
1384 case ZEND_FETCH_UNSET:
1385 case ZEND_UNSET_VAR:
1386 case ZEND_ISSET_ISEMPTY_VAR:
1387 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1388 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1389 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1390 !op_array->function_name) {
1391 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1392 }
1393 break;
1394 }
1395 }
1396 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1397 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1398 call_level++;
1399 stack_top += zend_jit_trace_frame_size(p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info));
1400 if (stack_top > stack_size) {
1401 stack_size = stack_top;
1402 }
1403 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1404 uint32_t num_args = 0;
1405 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1406 if (p->func
1407 && p->func != (zend_function*)&zend_pass_function
1408 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1409 || zend_string_equals_literal(p->func->common.function_name, "compact")
1410 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1411 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1412 }
1413 }
1414 if (!p->func) {
1415 /* Find num_args in the corresponding ZEND_JIT_TRACE_INIT_CALL record */
1416 num_args = find_trampoline_num_args(trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE, p);
1417 }
1418 frame_size = zend_jit_trace_frame_size(p->op_array, num_args);
1419 if (call_level == 0) {
1420 if (stack_top + frame_size > stack_size) {
1421 stack_size = stack_top + frame_size;
1422 }
1423 } else {
1424 call_level--;
1425 stack_top -= frame_size;
1426 }
1427 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1428 op_array = p->op_array;
1429 if (call_level == 0) {
1430 stack_top += zend_jit_trace_frame_size(op_array, 0);
1431 if (stack_top > stack_size) {
1432 stack_size = stack_top;
1433 }
1434 } else {
1435 call_level--;
1436 }
1437 level++;
1438 jit_extension =
1439 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1440 ssa = &jit_extension->func_info.ssa;
1441 if (ssa->cfg.blocks_count) {
1442 /* pass */
1443 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1444 /* Too many functions in single trace */
1445 *num_op_arrays_ptr = num_op_arrays;
1446 return NULL;
1447 } else {
1448 /* Remember op_array to cleanup */
1449 op_arrays[num_op_arrays++] = op_array;
1450 /* Build SSA */
1451 ssa = zend_jit_trace_build_ssa(op_array, script);
1452 }
1453 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1454 if (level == 0) {
1455 stack_bottom += zend_jit_trace_frame_size(p->op_array, 0);
1456 jit_extension =
1457 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1458 ssa = &jit_extension->func_info.ssa;
1459 if (ssa->cfg.blocks_count) {
1460 /* pass */
1461 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1462 /* Too many functions in single trace */
1463 *num_op_arrays_ptr = num_op_arrays;
1464 return NULL;
1465 } else {
1466 /* Remember op_array to cleanup */
1467 op_arrays[num_op_arrays++] = op_array;
1468 /* Build SSA */
1469 ssa = zend_jit_trace_build_ssa(op_array, script);
1470 }
1471 } else {
1472 stack_top -= zend_jit_trace_frame_size(op_array, 0);
1473 level--;
1474 }
1475 op_array = p->op_array;
1476 } else if (p->op == ZEND_JIT_TRACE_END) {
1477 break;
1478 }
1479 }
1480 *num_op_arrays_ptr = num_op_arrays;
1481
1482 /* Allocate space for abstract stack */
1483 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1484
1485 /* 2. Construct TSSA */
1486 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1487 tssa->cfg.flags = ZEND_SSA_TSSA;
1488 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1489 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1490 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1491
1492 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1493 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1494 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1495 tssa->cfg.blocks_count = 2;
1496 tssa->cfg.edges_count = 2;
1497
1498 tssa->cfg.predecessors[0] = 0;
1499 tssa->cfg.predecessors[1] = 1;
1500
1501 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1502 tssa->cfg.blocks[0].successors_count = 1;
1503 tssa->cfg.blocks[0].predecessors_count = 0;
1504 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1505 tssa->cfg.blocks[0].successors[0] = 1;
1506
1507 tssa->cfg.blocks[1].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1508 tssa->cfg.blocks[1].len = ssa_ops_count;
1509 tssa->cfg.blocks[1].successors_count = 1;
1510 tssa->cfg.blocks[1].predecessors_count = 2;
1511 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1512 tssa->cfg.blocks[1].successors[1] = 1;
1513 } else {
1514 tssa->cfg.blocks_count = 1;
1515 tssa->cfg.edges_count = 0;
1516
1517 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1518 tssa->cfg.blocks[0].len = ssa_ops_count;
1519 tssa->cfg.blocks[0].successors_count = 0;
1520 tssa->cfg.blocks[0].predecessors_count = 0;
1521 }
1522 ((zend_tssa*)tssa)->used_stack = -1;
1523
1524 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1525 return tssa;
1526 }
1527
1528 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1529 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1530 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1531 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1532 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1533
1534 op_array = trace_buffer->op_array;
1535 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1536 ssa_vars_count = op_array->last_var;
1537 } else {
1538 ssa_vars_count = op_array->last_var + op_array->T;
1539 }
1540 stack = frame->stack;
1541 for (i = 0; i < ssa_vars_count; i++) {
1542 SET_STACK_VAR(stack, i, i);
1543 }
1544
1545 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1546 // TODO: For tracing, it's possible, to create pseudo Phi functions
1547 // at the end of loop, without this additional pass (like LuaJIT) ???
1548 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1549 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1550 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1551 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1552 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1553 }
1554
1555 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1556 idx = 0;
1557 level = 0;
1558 for (;;p++) {
1559 if (p->op == ZEND_JIT_TRACE_VM) {
1560 opline = p->opline;
1561 ssa_opcodes[idx] = opline;
1562 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1563 idx++;
1564 len = zend_jit_trace_op_len(p->opline);
1565 while (len > 1) {
1566 opline++;
1567 ssa_opcodes[idx] = opline;
1568 if (opline->opcode != ZEND_OP_DATA) {
1569 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1570 }
1571 idx++;
1572 len--;
1573 }
1574 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1575 frame = zend_jit_trace_call_frame(frame, op_array, 0);
1576 stack = frame->stack;
1577 op_array = p->op_array;
1578 level++;
1579 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1580 return NULL;
1581 }
1582 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1583 for (i = 0; i < op_array->last_var; i++) {
1584 SET_STACK_VAR(stack, i, ssa_vars_count++);
1585 }
1586 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1587 op_array = p->op_array;
1588 frame = zend_jit_trace_ret_frame(frame, op_array);
1589 stack = frame->stack;
1590 if (level == 0) {
1591 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1592 return NULL;
1593 }
1594 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1595 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1596 SET_STACK_VAR(stack, i, ssa_vars_count++);
1597 }
1598 } else {
1599 level--;
1600 }
1601 } else if (p->op == ZEND_JIT_TRACE_END) {
1602 break;
1603 }
1604 }
1605
1606 op_array = trace_buffer->op_array;
1607 tssa->vars_count = ssa_vars_count;
1608 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1609 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1610 vars_count = op_array->last_var;
1611 } else {
1612 vars_count = op_array->last_var + op_array->T;
1613 }
1614 i = 0;
1615 while (i < vars_count) {
1616 ssa_vars[i].var = i;
1617 ssa_vars[i].scc = -1;
1618 ssa_vars[i].definition = -1;
1619 ssa_vars[i].use_chain = -1;
1620 i++;
1621 }
1622 while (i < tssa->vars_count) {
1623 ssa_vars[i].var = -1;
1624 ssa_vars[i].scc = -1;
1625 ssa_vars[i].definition = -1;
1626 ssa_vars[i].use_chain = -1;
1627 i++;
1628 }
1629
1630 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1631 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1632 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1633 /* Update Phi sources */
1634 zend_ssa_phi *phi = tssa->blocks[1].phis;
1635
1636 while (phi) {
1637 phi->sources[1] = STACK_VAR(stack, phi->var);
1638 ssa_vars[phi->ssa_var].var = phi->var;
1639 ssa_vars[phi->ssa_var].definition_phi = phi;
1640 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1641 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1642 phi = phi->next;
1643 }
1644 }
1645
1646 /* 3. Compute use-def chains */
1647 idx = (ssa_ops_count - 1);
1648 op = ssa_ops + idx;
1649 while (idx >= 0) {
1650 opline = ssa_opcodes[idx];
1651 if (op->op1_use >= 0) {
1652 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1653 ssa_vars[op->op1_use].use_chain = idx;
1654 }
1655 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1656 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1657 ssa_vars[op->op2_use].use_chain = idx;
1658 }
1659 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1660 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1661 ssa_vars[op->result_use].use_chain = idx;
1662 }
1663 if (op->op1_def >= 0) {
1664 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1665 ssa_vars[op->op1_def].definition = idx;
1666 }
1667 if (op->op2_def >= 0) {
1668 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1669 ssa_vars[op->op2_def].definition = idx;
1670 }
1671 if (op->result_def >= 0) {
1672 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1673 ssa_vars[op->result_def].definition = idx;
1674 }
1675 op--;
1676 idx--;
1677 }
1678
1679 /* 4. Type inference */
1680 op_array = trace_buffer->op_array;
1681 opline = trace_buffer[1].opline;
1682 jit_extension =
1683 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1684 ssa = &jit_extension->func_info.ssa;
1685
1686 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1687
1688 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1689 i = 0;
1690 while (i < op_array->last_var) {
1691 if (i < op_array->num_args) {
1692 if (ssa->var_info
1693 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i, NULL)) {
1694 /* pass */
1695 } else {
1696 if (ssa->vars) {
1697 ssa_vars[i].no_val = ssa->vars[i].no_val;
1698 ssa_vars[i].alias = ssa->vars[i].alias;
1699 } else {
1700 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1701 }
1702 if (op_array->arg_info && i < trace_buffer[1].opline - op_array->opcodes) {
1703 zend_arg_info *arg_info = &op_array->arg_info[i];
1704 zend_class_entry *ce;
1705 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1706
1707 if (ZEND_ARG_SEND_MODE(arg_info)) {
1708 tmp |= MAY_BE_REF;
1709 }
1710 ssa_var_info[i].type = tmp;
1711 ssa_var_info[i].ce = ce;
1712 ssa_var_info[i].is_instanceof = 1;
1713 } else {
1714 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1715 }
1716 }
1717 } else {
1718 if (ssa->vars) {
1719 ssa_vars[i].no_val = ssa->vars[i].no_val;
1720 ssa_vars[i].alias = ssa->vars[i].alias;
1721 } else {
1722 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1723 }
1724 if (ssa_vars[i].alias == NO_ALIAS) {
1725 ssa_var_info[i].type = MAY_BE_UNDEF;
1726 } else {
1727 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1728 }
1729 }
1730 i++;
1731 }
1732 } else {
1733 int parent_vars_count = 0;
1734 zend_jit_trace_stack *parent_stack = NULL;
1735
1736 i = 0;
1737 if (parent_trace) {
1738 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1739 op_array->last_var + op_array->T);
1740 if (parent_vars_count) {
1741 parent_stack =
1742 zend_jit_traces[parent_trace].stack_map +
1743 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1744 }
1745 }
1746 while (i < op_array->last_var + op_array->T) {
1747 if (!ssa->var_info
1748 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i, opline)) {
1749 if (ssa->vars && i < ssa->vars_count) {
1750 ssa_vars[i].alias = ssa->vars[i].alias;
1751 } else {
1752 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1753 }
1754 if (i < op_array->last_var) {
1755 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1756 } else {
1757 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1758 }
1759 }
1760 if (i < parent_vars_count) {
1761 /* Initialize TSSA variable from parent trace */
1762 uint8_t op_type = STACK_TYPE(parent_stack, i);
1763
1764 if (op_type != IS_UNKNOWN) {
1765 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1766 if (!ssa_var_info[i].type
1767 && op_type == IS_UNDEF
1768 && i >= op_array->last_var) {
1769 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1770 ssa_var_info[i].type |= MAY_BE_UNDEF;
1771 }
1772 }
1773 }
1774 i++;
1775 }
1776 }
1777
1778 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1779 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1780 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1781 /* Propagate initial value through Phi functions */
1782 zend_ssa_phi *phi = tssa->blocks[1].phis;
1783
1784 while (phi) {
1785 if (!ssa->var_info
1786 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var, NULL)) {
1787 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1788 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1789 }
1790 phi = phi->next;
1791 }
1792 }
1793
1794 frame = JIT_G(current_frame);
1795 top = zend_jit_trace_call_frame(frame, op_array, 0);
1796 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1797 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1798 frame->used_stack = 0;
1799 memset(&return_value_info, 0, sizeof(return_value_info));
1800
1801 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1802 max_used_stack = used_stack = 0;
1803 } else {
1804 max_used_stack = used_stack = -1;
1805 }
1806
1807 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1808 idx = 0;
1809 level = 0;
1810 opline = NULL;
1811 for (;;p++) {
1812 if (p->op == ZEND_JIT_TRACE_VM) {
1813 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1814 uint8_t val_type = IS_UNKNOWN;
1815 // zend_class_entry *op1_ce = NULL;
1816 zend_class_entry *op2_ce = NULL;
1817
1818 opline = p->opline;
1819
1820 op1_type = orig_op1_type = p->op1_type;
1821 op2_type = orig_op2_type = p->op2_type;
1822 op3_type = p->op3_type;
1823 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1824 op1_type = IS_UNKNOWN;
1825 }
1826 if (op1_type != IS_UNKNOWN) {
1827 op1_type &= ~IS_TRACE_PACKED;
1828 }
1829 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1830 op2_type = IS_UNKNOWN;
1831 }
1832 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1833 op3_type = IS_UNKNOWN;
1834 }
1835
1836 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1837 // op1_ce = (zend_class_entry*)(p+1)->ce;
1838 p++;
1839 }
1840 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1841 op2_ce = (zend_class_entry*)(p+1)->ce;
1842 p++;
1843 }
1844 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1845 val_type = (p+1)->op1_type;
1846 p++;
1847 }
1848
1849 switch (opline->opcode) {
1850 case ZEND_ASSIGN_OP:
1851 if (opline->extended_value == ZEND_POW
1852 || opline->extended_value == ZEND_DIV) {
1853 // TODO: check for division by zero ???
1854 break;
1855 }
1856 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1857 break;
1858 }
1859 ADD_OP1_TRACE_GUARD();
1860 ADD_OP2_TRACE_GUARD();
1861 break;
1862 case ZEND_ASSIGN_DIM_OP:
1863 if (opline->result_type != IS_UNUSED) {
1864 break;
1865 }
1866 if (op3_type != IS_UNKNOWN
1867 && !zend_jit_supported_binary_op(
1868 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1869 break;
1870 }
1871 ZEND_FALLTHROUGH;
1872 case ZEND_ASSIGN_DIM:
1873 if (opline->opcode == ZEND_ASSIGN_DIM
1874 && opline->op1_type == IS_CV
1875 && (opline+1)->op1_type == IS_CV
1876 && (opline+1)->op1.var == opline->op1.var) {
1877 /* skip $a[x] = $a; */
1878 break;
1879 }
1880 if (opline->op1_type == IS_CV || opline->opcode == ZEND_ASSIGN_DIM_OP) {
1881 ADD_OP1_DATA_TRACE_GUARD();
1882 }
1883 ADD_OP2_TRACE_GUARD();
1884 ADD_OP1_TRACE_GUARD();
1885 if (op1_type == IS_ARRAY
1886 && ((opline->op2_type == IS_CONST
1887 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1888 || (opline->op2_type != IS_CONST
1889 && op2_type == IS_LONG))) {
1890
1891 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1892 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1893
1894 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1895 info->type |= MAY_BE_PACKED_GUARD;
1896 info->type &= ~MAY_BE_ARRAY_PACKED;
1897 }
1898 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1899 && val_type != IS_UNKNOWN
1900 && val_type != IS_UNDEF) {
1901 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1902
1903 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1904 info->type |= MAY_BE_PACKED_GUARD;
1905 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1906 }
1907 }
1908 }
1909 break;
1910 case ZEND_ASSIGN_OBJ_OP:
1911 if (opline->extended_value == ZEND_POW
1912 || opline->extended_value == ZEND_DIV) {
1913 // TODO: check for division by zero ???
1914 break;
1915 }
1916 if (opline->result_type != IS_UNUSED) {
1917 break;
1918 }
1919 ZEND_FALLTHROUGH;
1920 case ZEND_ASSIGN_OBJ:
1921 case ZEND_PRE_INC_OBJ:
1922 case ZEND_PRE_DEC_OBJ:
1923 case ZEND_POST_INC_OBJ:
1924 case ZEND_POST_DEC_OBJ:
1925 if (opline->op2_type != IS_CONST
1926 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1927 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1928 break;
1929 }
1930 if (opline->opcode == ZEND_ASSIGN_OBJ
1931 || opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1932 if (opline->op1_type == IS_CV
1933 && (opline+1)->op1_type == IS_CV
1934 && (opline+1)->op1.var == opline->op1.var) {
1935 /* skip $a->prop += $a; */
1936 break;
1937 }
1938 ADD_OP1_DATA_TRACE_GUARD();
1939 }
1940 ADD_OP1_TRACE_GUARD();
1941 break;
1942 case ZEND_CONCAT:
1943 case ZEND_FAST_CONCAT:
1944 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1945 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1946 ADD_OP2_TRACE_GUARD();
1947 ADD_OP1_TRACE_GUARD();
1948 }
1949 break;
1950 case ZEND_ADD:
1951 case ZEND_SUB:
1952 case ZEND_MUL:
1953 // case ZEND_DIV: // TODO: check for division by zero ???
1954 if (orig_op1_type == IS_UNDEF || orig_op2_type == IS_UNDEF) {
1955 break;
1956 }
1957 ZEND_FALLTHROUGH;
1958 case ZEND_IS_EQUAL:
1959 case ZEND_IS_NOT_EQUAL:
1960 case ZEND_IS_SMALLER:
1961 case ZEND_IS_SMALLER_OR_EQUAL:
1962 case ZEND_CASE:
1963 case ZEND_IS_IDENTICAL:
1964 case ZEND_IS_NOT_IDENTICAL:
1965 case ZEND_CASE_STRICT:
1966 case ZEND_BW_OR:
1967 case ZEND_BW_AND:
1968 case ZEND_BW_XOR:
1969 case ZEND_SL:
1970 case ZEND_SR:
1971 case ZEND_MOD:
1972 ADD_OP2_TRACE_GUARD();
1973 ZEND_FALLTHROUGH;
1974 case ZEND_ECHO:
1975 case ZEND_STRLEN:
1976 case ZEND_COUNT:
1977 case ZEND_QM_ASSIGN:
1978 case ZEND_FE_RESET_R:
1979 ADD_OP1_TRACE_GUARD();
1980 break;
1981 case ZEND_FE_FETCH_R:
1982 ADD_OP1_TRACE_GUARD();
1983 if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) {
1984
1985 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1986
1987 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1988 info->type |= MAY_BE_PACKED_GUARD;
1989 if (orig_op1_type & IS_TRACE_PACKED) {
1990 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1991 } else {
1992 info->type &= ~MAY_BE_ARRAY_PACKED;
1993 }
1994 }
1995 }
1996 break;
1997 case ZEND_VERIFY_RETURN_TYPE:
1998 if (opline->op1_type == IS_UNUSED) {
1999 /* Always throws */
2000 break;
2001 }
2002 if (opline->op1_type == IS_CONST) {
2003 /* TODO Different instruction format, has return value */
2004 break;
2005 }
2006 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
2007 /* Not worth bothering with */
2008 break;
2009 }
2010 ADD_OP1_TRACE_GUARD();
2011 break;
2012 case ZEND_FETCH_DIM_FUNC_ARG:
2013 if (!frame
2014 || !frame->call
2015 || !frame->call->func
2016 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2017 break;
2018 }
2019 ADD_OP2_TRACE_GUARD();
2020 ADD_OP1_TRACE_GUARD();
2021 break;
2022 case ZEND_PRE_INC:
2023 case ZEND_PRE_DEC:
2024 case ZEND_POST_INC:
2025 case ZEND_POST_DEC:
2026 if (opline->op1_type != IS_CV) {
2027 break;
2028 }
2029 ADD_OP1_TRACE_GUARD();
2030 break;
2031 case ZEND_ASSIGN:
2032 if (opline->op1_type != IS_CV) {
2033 break;
2034 }
2035 ADD_OP2_TRACE_GUARD();
2036 if (op1_type != IS_UNKNOWN
2037 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
2038 ADD_OP1_TRACE_GUARD();
2039 }
2040 break;
2041 case ZEND_CAST:
2042 if (opline->extended_value != op1_type) {
2043 break;
2044 }
2045 ADD_OP1_TRACE_GUARD();
2046 break;
2047 case ZEND_JMPZ:
2048 case ZEND_JMPNZ:
2049 case ZEND_JMPZ_EX:
2050 case ZEND_JMPNZ_EX:
2051 case ZEND_BOOL:
2052 case ZEND_BOOL_NOT:
2053 ADD_OP1_TRACE_GUARD();
2054 break;
2055 case ZEND_ISSET_ISEMPTY_CV:
2056 if ((opline->extended_value & ZEND_ISEMPTY)) {
2057 // TODO: support for empty() ???
2058 break;
2059 }
2060 ADD_OP1_TRACE_GUARD();
2061 break;
2062 case ZEND_IN_ARRAY:
2063 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
2064 break;
2065 }
2066 ADD_OP1_TRACE_GUARD();
2067 break;
2068 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
2069 if ((opline->extended_value & ZEND_ISEMPTY)) {
2070 // TODO: support for empty() ???
2071 break;
2072 }
2073 ZEND_FALLTHROUGH;
2074 case ZEND_FETCH_DIM_R:
2075 case ZEND_FETCH_DIM_IS:
2076 case ZEND_FETCH_LIST_R:
2077 ADD_OP1_TRACE_GUARD();
2078 ADD_OP2_TRACE_GUARD();
2079
2080 if (op1_type == IS_ARRAY
2081 && opline->op1_type != IS_CONST
2082 && ((opline->op2_type == IS_CONST
2083 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2084 || (opline->op2_type != IS_CONST
2085 && op2_type == IS_LONG))) {
2086
2087 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
2088
2089 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
2090 info->type |= MAY_BE_PACKED_GUARD;
2091 if (orig_op1_type & IS_TRACE_PACKED) {
2092 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2093 } else {
2094 info->type &= ~MAY_BE_ARRAY_PACKED;
2095 }
2096 }
2097 }
2098 break;
2099 case ZEND_FETCH_DIM_W:
2100 case ZEND_FETCH_DIM_RW:
2101 // case ZEND_FETCH_DIM_UNSET:
2102 case ZEND_FETCH_LIST_W:
2103 if (opline->op1_type != IS_CV
2104 && (orig_op1_type == IS_UNKNOWN
2105 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
2106 break;
2107 }
2108 ADD_OP1_TRACE_GUARD();
2109 ADD_OP2_TRACE_GUARD();
2110 if (op1_type == IS_ARRAY
2111 && !(orig_op1_type & IS_TRACE_PACKED)
2112 && ((opline->op2_type == IS_CONST
2113 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2114 || (opline->op2_type != IS_CONST
2115 && op2_type == IS_LONG))) {
2116
2117 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
2118
2119 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
2120 info->type |= MAY_BE_PACKED_GUARD;
2121 info->type &= ~MAY_BE_ARRAY_PACKED;
2122 }
2123 }
2124 break;
2125 case ZEND_SEND_VAL_EX:
2126 case ZEND_SEND_VAR_EX:
2127 case ZEND_SEND_VAR_NO_REF_EX:
2128 if (opline->op2_type == IS_CONST) {
2129 /* Named parameters not supported in JIT */
2130 break;
2131 }
2132 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
2133 goto propagate_arg;
2134 }
2135 ZEND_FALLTHROUGH;
2136 case ZEND_SEND_VAL:
2137 case ZEND_SEND_VAR:
2138 case ZEND_SEND_VAR_NO_REF:
2139 case ZEND_SEND_FUNC_ARG:
2140 if (opline->op2_type == IS_CONST) {
2141 /* Named parameters not supported in JIT */
2142 break;
2143 }
2144 ADD_OP1_TRACE_GUARD();
2145 propagate_arg:
2146 /* Propagate argument type */
2147 if (frame->call
2148 && frame->call->func
2149 && frame->call->func->type == ZEND_USER_FUNCTION
2150 && opline->op2.num <= frame->call->func->op_array.num_args) {
2151 uint32_t info;
2152
2153 if (opline->op1_type == IS_CONST) {
2154 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
2155 } else {
2156 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2157 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
2158 }
2159 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
2160 zend_arg_info *arg_info;
2161
2162 ZEND_ASSERT(frame->call->func->op_array.arg_info);
2163 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
2164 if (ZEND_TYPE_IS_SET(arg_info->type)) {
2165 zend_class_entry *ce;
2166 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2167 info &= tmp;
2168 if (!info) {
2169 break;
2170 }
2171 }
2172 }
2173 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2174 info |= MAY_BE_RCN;
2175 }
2176 if (info & MAY_BE_UNDEF) {
2177 info |= MAY_BE_NULL;
2178 info &= ~MAY_BE_UNDEF;
2179 }
2180 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2181 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2182 }
2183 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2184 }
2185 break;
2186 case ZEND_RETURN:
2187 ADD_OP1_TRACE_GUARD();
2188 /* Propagate return value types */
2189 if (opline->op1_type == IS_UNUSED) {
2190 return_value_info.type = MAY_BE_NULL;
2191 } else if (opline->op1_type == IS_CONST) {
2192 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2193 } else {
2194 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2195 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2196 if (return_value_info.type & MAY_BE_UNDEF) {
2197 return_value_info.type &= ~MAY_BE_UNDEF;
2198 return_value_info.type |= MAY_BE_NULL;
2199 }
2200 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2201 /* CVs are going to be destructed and the reference-counter
2202 of return value may be decremented to 1 */
2203 return_value_info.type |= MAY_BE_RC1;
2204 }
2205 return_value_info.type &= ~MAY_BE_GUARD;
2206 }
2207 break;
2208 case ZEND_CHECK_FUNC_ARG:
2209 if (!frame
2210 || !frame->call
2211 || !frame->call->func) {
2212 break;
2213 }
2214 if (opline->op2_type == IS_CONST
2215 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2216 /* Named parameters not supported in JIT */
2217 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2218 break;
2219 }
2220 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2221 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2222 } else {
2223 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2224 }
2225 break;
2226 case ZEND_FETCH_OBJ_FUNC_ARG:
2227 if (!frame
2228 || !frame->call
2229 || !frame->call->func
2230 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2231 break;
2232 }
2233 ZEND_FALLTHROUGH;
2234 case ZEND_FETCH_OBJ_R:
2235 case ZEND_FETCH_OBJ_IS:
2236 case ZEND_FETCH_OBJ_W:
2237 if (opline->op2_type != IS_CONST
2238 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2239 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2240 break;
2241 }
2242 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2243 ADD_OP1_TRACE_GUARD();
2244 }
2245 break;
2246 case ZEND_INIT_METHOD_CALL:
2247 if (opline->op2_type != IS_CONST
2248 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2249 break;
2250 }
2251 ADD_OP1_TRACE_GUARD();
2252 break;
2253 case ZEND_INIT_DYNAMIC_CALL:
2254 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2255 ADD_OP2_TRACE_GUARD();
2256 }
2257 break;
2258 case ZEND_SEND_ARRAY:
2259 case ZEND_SEND_UNPACK:
2260 case ZEND_CHECK_UNDEF_ARGS:
2261 case ZEND_INCLUDE_OR_EVAL:
2262 max_used_stack = used_stack = -1;
2263 break;
2264 case ZEND_TYPE_CHECK:
2265 if (opline->extended_value == MAY_BE_RESOURCE) {
2266 // TODO: support for is_resource() ???
2267 break;
2268 }
2269 if (op1_type != IS_UNKNOWN
2270 && (opline->extended_value == (1 << op1_type)
2271 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2272 /* add guards only for exact checks, to avoid code duplication */
2273 ADD_OP1_TRACE_GUARD();
2274 }
2275 break;
2276 case ZEND_ROPE_INIT:
2277 case ZEND_ROPE_ADD:
2278 case ZEND_ROPE_END:
2279 if (op2_type == IS_STRING) {
2280 ADD_OP2_TRACE_GUARD();
2281 }
2282 break;
2283 default:
2284 break;
2285 }
2286 len = zend_jit_trace_op_len(opline);
2287 if (ssa->var_info) {
2288 /* Add statically inferred ranges */
2289 if (ssa_ops[idx].op1_def >= 0) {
2290 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2291 }
2292 if (ssa_ops[idx].op2_def >= 0) {
2293 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2294 }
2295 if (ssa_ops[idx].result_def >= 0) {
2296 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2297 }
2298 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2299 if (ssa_ops[idx+1].op1_def >= 0) {
2300 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2301 }
2302 if (ssa_ops[idx+1].op2_def >= 0) {
2303 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2304 }
2305 if (ssa_ops[idx+1].result_def >= 0) {
2306 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2307 }
2308 }
2309 } else {
2310 if (ssa_ops[idx].op1_def >= 0) {
2311 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2312 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2313 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2314 }
2315 }
2316 if (ssa_ops[idx].op2_def >= 0) {
2317 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2318 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2319 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2320 }
2321 }
2322 if (ssa_ops[idx].result_def >= 0) {
2323 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2324 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2325 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2326 }
2327 }
2328 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2329 if (ssa_ops[idx+1].op1_def >= 0) {
2330 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2331 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2332 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2333 }
2334 }
2335 if (ssa_ops[idx+1].op2_def >= 0) {
2336 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2337 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2338 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2339 }
2340 }
2341 if (ssa_ops[idx+1].result_def >= 0) {
2342 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2343 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2344 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2345 }
2346 }
2347 }
2348 }
2349 if (opline->opcode == ZEND_RECV_INIT
2350 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2351 /* RECV_INIT always copy the constant */
2352 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2353 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2354 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2355 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2356 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2357 }
2358 } else {
2359 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2360 // TODO:
2361 assert(0);
2362 }
2363 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2364 && ssa_ops[idx].op1_def >= 0
2365 && op1_type == IS_ARRAY
2366 && (orig_op1_type & IS_TRACE_PACKED)
2367 && val_type != IS_UNKNOWN
2368 && val_type != IS_UNDEF
2369 && ((opline->op2_type == IS_CONST
2370 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2371 || (opline->op2_type != IS_CONST
2372 && op2_type == IS_LONG))) {
2373 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2374
2375 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2376 }
2377 }
2378 if (ssa->var_info) {
2379 /* Add statically inferred restrictions */
2380 if (ssa_ops[idx].op1_def >= 0) {
2381 if (opline->opcode == ZEND_SEND_VAR_EX
2382 && frame
2383 && frame->call
2384 && frame->call->func
2385 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2386 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2387 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2388 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2389 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2390 }
2391 } else {
2392 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2393 }
2394 }
2395 if (ssa_ops[idx].op2_def >= 0) {
2396 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2397 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2398 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2399 }
2400 }
2401 if (ssa_ops[idx].result_def >= 0) {
2402 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2403 }
2404 }
2405 idx++;
2406 while (len > 1) {
2407 opline++;
2408 if (opline->opcode != ZEND_OP_DATA) {
2409 if (ssa->var_info) {
2410 /* Add statically inferred ranges */
2411 if (ssa_ops[idx].op1_def >= 0) {
2412 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2413 }
2414 if (ssa_ops[idx].op2_def >= 0) {
2415 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2416 }
2417 if (ssa_ops[idx].result_def >= 0) {
2418 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2419 }
2420 } else {
2421 if (ssa_ops[idx].op1_def >= 0) {
2422 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2423 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2424 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2425 }
2426 }
2427 if (ssa_ops[idx].op2_def >= 0) {
2428 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2429 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2430 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2431 }
2432 }
2433 if (ssa_ops[idx].result_def >= 0) {
2434 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2435 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2436 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2437 }
2438 }
2439 }
2440 if (opline->opcode == ZEND_RECV_INIT
2441 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2442 /* RECV_INIT always copy the constant */
2443 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2444 } else {
2445 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2446 // TODO:
2447 assert(0);
2448 }
2449 }
2450 }
2451 if (ssa->var_info) {
2452 /* Add statically inferred restrictions */
2453 if (ssa_ops[idx].op1_def >= 0) {
2454 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2455 }
2456 if (ssa_ops[idx].op2_def >= 0) {
2457 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2458 }
2459 if (ssa_ops[idx].result_def >= 0) {
2460 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2461 }
2462 }
2463 idx++;
2464 len--;
2465 }
2466
2467 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2468 op_array = p->op_array;
2469 jit_extension =
2470 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2471 ssa = &jit_extension->func_info.ssa;
2472
2473 call = frame->call;
2474 if (!call) {
2475 /* Trace missed INIT_FCALL opcode */
2476 call = top;
2477 TRACE_FRAME_INIT(call, op_array, 0, 0);
2478 call->used_stack = 0;
2479 top = zend_jit_trace_call_frame(top, op_array, 0);
2480 } else {
2481 ZEND_ASSERT(&call->func->op_array == op_array);
2482 }
2483 frame->call = call->prev;
2484 call->prev = frame;
2485 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2486 frame = call;
2487
2488 level++;
2489 i = 0;
2490 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2491 while (i < op_array->last_var) {
2492 ssa_vars[v].var = i;
2493 if (i < op_array->num_args) {
2494 if (ssa->var_info
2495 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2496 /* pass */
2497 } else {
2498 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2499 if (op_array->arg_info) {
2500 zend_arg_info *arg_info = &op_array->arg_info[i];
2501 zend_class_entry *ce;
2502 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2503
2504 if (ZEND_ARG_SEND_MODE(arg_info)) {
2505 tmp |= MAY_BE_REF;
2506 }
2507 ssa_var_info[v].type = tmp;
2508 ssa_var_info[v].ce = ce;
2509 ssa_var_info[v].is_instanceof = 1;
2510 } else {
2511 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2512 }
2513 }
2514 } else {
2515 if (ssa->vars) {
2516 ssa_vars[v].no_val = ssa->vars[i].no_val;
2517 ssa_vars[v].alias = ssa->vars[i].alias;
2518 } else {
2519 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2520 }
2521 if (ssa_vars[v].alias == NO_ALIAS) {
2522 ssa_var_info[v].type = MAY_BE_UNDEF;
2523 } else {
2524 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2525 }
2526 }
2527 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2528 && i < op_array->num_args) {
2529 /* Propagate argument type */
2530 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2531 }
2532 i++;
2533 v++;
2534 }
2535 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2536 op_array = p->op_array;
2537 jit_extension =
2538 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2539 ssa = &jit_extension->func_info.ssa;
2540 if (level == 0) {
2541 i = 0;
2542 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2543 while (i < op_array->last_var) {
2544 ssa_vars[v].var = i;
2545 if (!ssa->var_info
2546 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2547 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2548 }
2549 i++;
2550 v++;
2551 }
2552 while (i < op_array->last_var + op_array->T) {
2553 ssa_vars[v].var = i;
2554 if (!ssa->var_info
2555 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2556 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2557 }
2558 i++;
2559 v++;
2560 }
2561 if (return_value_info.type != 0) {
2562 zend_jit_trace_rec *q = p + 1;
2563 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2564 q++;
2565 }
2566 if (q->op == ZEND_JIT_TRACE_VM
2567 || (q->op == ZEND_JIT_TRACE_END
2568 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2569 const zend_op *opline = q->opline - 1;
2570 if (opline->result_type != IS_UNUSED) {
2571 ssa_var_info[
2572 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2573 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2574 }
2575 }
2576 memset(&return_value_info, 0, sizeof(return_value_info));
2577 }
2578 } else {
2579 level--;
2580 if (return_value_info.type != 0) {
2581 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2582 const zend_op *opline = (p+1)->opline - 1;
2583 if (opline->result_type != IS_UNUSED) {
2584 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2585 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2586 }
2587 }
2588 }
2589 memset(&return_value_info, 0, sizeof(return_value_info));
2590 }
2591 }
2592
2593 top = frame;
2594 if (frame->prev) {
2595 if (used_stack > 0) {
2596 used_stack -= frame->used_stack;
2597 }
2598 frame = frame->prev;
2599 ZEND_ASSERT(&frame->func->op_array == op_array);
2600 } else {
2601 max_used_stack = used_stack = -1;
2602 frame = zend_jit_trace_ret_frame(frame, op_array);
2603 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2604 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2605 frame->used_stack = 0;
2606 }
2607
2608 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2609 call = top;
2610 TRACE_FRAME_INIT(call, p->func, 0, 0);
2611 call->prev = frame->call;
2612 call->used_stack = 0;
2613 frame->call = call;
2614 top = zend_jit_trace_call_frame(top, p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info));
2615 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2616 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2617 SET_STACK_INFO(call->stack, i, -1);
2618 }
2619 }
2620 if (used_stack >= 0
2621 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2622 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2623 max_used_stack = used_stack = -1;
2624 } else {
2625 const zend_op *opline = (p-1)->opline;
2626
2627 switch (opline->opcode) {
2628 case ZEND_INIT_FCALL:
2629 case ZEND_INIT_FCALL_BY_NAME:
2630 case ZEND_INIT_NS_FCALL_BY_NAME:
2631 case ZEND_INIT_METHOD_CALL:
2632 case ZEND_INIT_DYNAMIC_CALL:
2633 case ZEND_INIT_STATIC_METHOD_CALL:
2634 //case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
2635 //case ZEND_INIT_USER_CALL:
2636 //case ZEND_NEW:
2637 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2638 used_stack += frame->used_stack;
2639 if (used_stack > max_used_stack) {
2640 max_used_stack = used_stack;
2641 }
2642 break;
2643 default:
2644 max_used_stack = used_stack = -1;
2645 }
2646 }
2647 }
2648 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2649 call = frame->call;
2650 if (call) {
2651 top = call;
2652 frame->call = call->prev;
2653 }
2654
2655 if (idx > 0
2656 && ssa_ops[idx-1].result_def >= 0
2657 && p->func
2658 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2659 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2660 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2661 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2662 opline->opcode == ZEND_DO_FCALL ||
2663 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2664
2665 if (opline->result_type != IS_UNDEF) {
2666 zend_class_entry *ce;
2667 const zend_function *func = p->func;
2668 zend_arg_info *ret_info = func->common.arg_info - 1;
2669 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2670
2671 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2672 }
2673 }
2674 } else if (p->op == ZEND_JIT_TRACE_END) {
2675 break;
2676 }
2677 }
2678
2679 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2680
2681 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2682 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2683 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2684 /* Propagate guards through Phi sources */
2685 zend_ssa_phi *phi = tssa->blocks[1].phis;
2686
2687 op_array = trace_buffer->op_array;
2688 jit_extension =
2689 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2690 ssa = &jit_extension->func_info.ssa;
2691
2692 while (phi) {
2693 uint32_t t = ssa_var_info[phi->ssa_var].type;
2694
2695 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2696 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2697 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2698
2699 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2700 if (!((t0 | t1) & MAY_BE_GUARD)) {
2701 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2702 }
2703 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2704 if (!(t1 & MAY_BE_GUARD)
2705 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2706 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2707 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2708 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2709 MAY_BE_GUARD;
2710 if (!(t0 & MAY_BE_ARRAY)) {
2711 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2712 }
2713 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2714 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2715 }
2716 ssa_var_info[phi->sources[0]].type = t0;
2717 ssa_var_info[phi->sources[0]].type = t0;
2718 }
2719 } else {
2720 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2721 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2722 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2723 MAY_BE_GUARD;
2724 if (!(t0 & MAY_BE_ARRAY)) {
2725 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2726 }
2727 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2728 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2729 }
2730 ssa_var_info[phi->sources[0]].type = t0;
2731 }
2732 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2733 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2734 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2735 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2736 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2737 MAY_BE_GUARD;
2738 if (!(t1 & MAY_BE_ARRAY)) {
2739 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2740 }
2741 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2742 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2743 }
2744 ssa_var_info[phi->sources[1]].type = t1;
2745 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2746 }
2747 }
2748 }
2749 t = ssa_var_info[phi->ssa_var].type;
2750 }
2751
2752 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2753 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2754 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2755
2756 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2757 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2758 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2759 }
2760 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2761 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2762 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2763 ssa_var_info[phi->sources[0]].type =
2764 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2765 }
2766 }
2767 }
2768 phi = phi->next;
2769 }
2770 }
2771
2772 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2773 if (parent_trace) {
2774 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2775 ZEND_JIT_TRACE_NUM,
2776 parent_trace,
2777 exit_num,
2778 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2779 trace_buffer->op_array->scope ? "::" : "",
2780 trace_buffer->op_array->function_name ?
2781 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2782 ZSTR_VAL(trace_buffer->op_array->filename),
2783 trace_buffer[1].opline->lineno);
2784 } else {
2785 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2786 ZEND_JIT_TRACE_NUM,
2787 zend_jit_trace_star_desc(trace_buffer->start),
2788 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2789 trace_buffer->op_array->scope ? "::" : "",
2790 trace_buffer->op_array->function_name ?
2791 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2792 ZSTR_VAL(trace_buffer->op_array->filename),
2793 trace_buffer[1].opline->lineno);
2794 }
2795 zend_jit_dump_trace(trace_buffer, tssa);
2796 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2797 uint32_t idx = trace_buffer[1].last;
2798 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2799 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2800 ZEND_JIT_TRACE_NUM,
2801 link_to);
2802 } else {
2803 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2804 ZEND_JIT_TRACE_NUM,
2805 zend_jit_trace_stop_description[trace_buffer->stop]);
2806 }
2807 }
2808
2809 return tssa;
2810 }
2811
2812 #define RA_HAS_IVAL(var) (ra[var].ref != 0)
2813 #define RA_IVAL_FLAGS(var) ra[var].flags
2814 #define RA_IVAL_START(var, line) do {ra[var].ref = IR_NULL;} while (0)
2815 #define RA_IVAL_END(var, line)
2816 #define RA_IVAL_CLOSE(var, line)
2817 #define RA_IVAL_DEL(var) do {ra[var].ref = IR_UNUSED;} while (0)
2818 #define RA_HAS_REG(var) (ra[var].ref != 0)
2819 #define RA_REG_FLAGS(var) ra[var].flags
2820 #define RA_REG_START(var, line) do {ra[var].ref = IR_NULL;} while (0)
2821 #define RA_REG_DEL(var) do {ra[var].ref = IR_UNUSED;} while (0)
2822
zend_jit_trace_use_var(int line,int var,int def,int use_chain,zend_jit_reg_var * ra,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2823 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, zend_jit_reg_var *ra, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2824 {
2825 ZEND_ASSERT(RA_HAS_IVAL(var));
2826 ZEND_ASSERT(!(RA_IVAL_FLAGS(var) & ZREG_LAST_USE));
2827 RA_IVAL_END(var, line);
2828 if (def >= 0) {
2829 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2830 } else if (use_chain < 0 && (RA_IVAL_FLAGS(var) & (ZREG_LOAD|ZREG_STORE))) {
2831 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2832 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2833 /* pass */
2834 } else if (op_array_ssa->vars) {
2835 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2836
2837 if (ssa->ops[line].op1_use == var) {
2838 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2839 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2840 }
2841 } else if (ssa->ops[line].op2_use == var) {
2842 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2843 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2844 }
2845 } else if (ssa->ops[line].result_use == var) {
2846 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2847 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2848 }
2849 }
2850 }
2851 }
2852
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2853 static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2854 {
2855 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2856 zend_jit_trace_rec *p;
2857 const zend_op_array *op_array;
2858 zend_jit_op_array_trace_extension *jit_extension;
2859 const zend_ssa *op_array_ssa;
2860 const zend_ssa_op *ssa_op;
2861 int i, j, idx, count, level;
2862 zend_jit_reg_var *ra;
2863 const zend_op_array **vars_op_array;
2864 void *checkpoint;
2865 zend_jit_trace_stack_frame *frame;
2866 zend_jit_trace_stack *stack;
2867 uint32_t parent_vars_count = parent_trace ?
2868 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2869 zend_jit_trace_stack *parent_stack = parent_vars_count ?
2870 zend_jit_traces[parent_trace].stack_map +
2871 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2872 checkpoint = zend_arena_checkpoint(CG(arena));
2873 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
2874 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2875 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2876
2877 op_array = trace_buffer->op_array;
2878 jit_extension =
2879 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2880 op_array_ssa = &jit_extension->func_info.ssa;
2881 frame = JIT_G(current_frame);
2882 frame->prev = NULL;
2883 frame->func = (const zend_function*)op_array;
2884 stack = frame->stack;
2885
2886 count = 0;
2887
2888 i = 0;
2889 j = op_array->last_var;
2890 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2891 j += op_array->T;
2892 }
2893 while (i < j) {
2894 SET_STACK_VAR(stack, i, i);
2895 vars_op_array[i] = op_array;
2896 /* We don't start intervals for variables used in Phi */
2897 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2898 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2899 && ssa->vars[i].alias == NO_ALIAS
2900 && zend_jit_var_supports_reg(ssa, i)) {
2901 RA_IVAL_START(i, 0);
2902 if (i < parent_vars_count
2903 && STACK_REG(parent_stack, i) != ZREG_NONE
2904 && STACK_FLAGS(parent_stack, i) != ZREG_ZVAL_COPY
2905 ) {
2906 /* We will try to reuse register from parent trace */
2907 RA_IVAL_FLAGS(i) = STACK_FLAGS(parent_stack, i);
2908 count += 2;
2909 } else {
2910 RA_IVAL_FLAGS(i) = ZREG_LOAD;
2911 count++;
2912 }
2913 }
2914 i++;
2915 }
2916
2917 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2918 j = op_array->last_var + op_array->T;
2919 while (i < j) {
2920 SET_STACK_VAR(stack, i, -1);
2921 i++;
2922 }
2923 }
2924
2925 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2926 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2927 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2928 zend_ssa_phi *phi = ssa->blocks[1].phis;
2929
2930 while (phi) {
2931 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2932 vars_op_array[phi->ssa_var] = op_array;
2933 if (ssa->vars[phi->ssa_var].use_chain >= 0
2934 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2935 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2936 RA_IVAL_START(phi->ssa_var, 0);
2937 count++;
2938 }
2939 phi = phi->next;
2940 }
2941 }
2942
2943 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2944 level = 0;
2945 ssa_op = ssa->ops;
2946 idx = 0;
2947 for (;;p++) {
2948 if (p->op == ZEND_JIT_TRACE_VM) {
2949 const zend_op *opline = p->opline;
2950 int len;
2951 bool support_opline;
2952
2953 support_opline =
2954 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2955
2956 if (support_opline
2957 && opline->opcode == ZEND_ASSIGN
2958 && opline->op1_type == IS_CV
2959 && ssa_op->op1_def >= 0
2960 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2961 /* avoid register allocation in case of possibility of indirect modification*/
2962 support_opline = 0;
2963 }
2964
2965 if (ssa_op->op1_use >= 0
2966 && RA_HAS_IVAL(ssa_op->op1_use)) {
2967 if (!support_opline) {
2968 RA_IVAL_DEL(ssa_op->op1_use);
2969 count--;
2970 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2971 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain,
2972 ra,
2973 ssa, ssa_opcodes, op_array, op_array_ssa);
2974 if (opline->op1_type != IS_CV) {
2975 if (opline->opcode == ZEND_CASE
2976 || opline->opcode == ZEND_CASE_STRICT
2977 || opline->opcode == ZEND_SWITCH_LONG
2978 || opline->opcode == ZEND_MATCH
2979 || opline->opcode == ZEND_FETCH_LIST_R
2980 || opline->opcode == ZEND_COPY_TMP
2981 || opline->opcode == ZEND_SWITCH_STRING
2982 || opline->opcode == ZEND_FE_FETCH_R
2983 || opline->opcode == ZEND_FE_FETCH_RW
2984 || opline->opcode == ZEND_FETCH_LIST_W
2985 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2986 || opline->opcode == ZEND_BIND_LEXICAL
2987 || opline->opcode == ZEND_ROPE_ADD) {
2988 /* The value is kept alive and may be used outside of the trace */
2989 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_STORE;
2990 } else {
2991 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_LAST_USE;
2992 }
2993 }
2994 }
2995 }
2996 if (ssa_op->op2_use >= 0
2997 && ssa_op->op2_use != ssa_op->op1_use
2998 && RA_HAS_IVAL(ssa_op->op2_use)) {
2999 if (!support_opline) {
3000 RA_IVAL_DEL(ssa_op->op2_use);
3001 count--;
3002 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
3003 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain,
3004 ra,
3005 ssa, ssa_opcodes, op_array, op_array_ssa);
3006 if (opline->op2_type != IS_CV) {
3007 RA_IVAL_FLAGS(ssa_op->op2_use) |= ZREG_LAST_USE;
3008 }
3009 }
3010 }
3011 if (ssa_op->result_use >= 0
3012 && ssa_op->result_use != ssa_op->op1_use
3013 && ssa_op->result_use != ssa_op->op2_use
3014 && RA_HAS_IVAL(ssa_op->result_use)) {
3015 if (!support_opline) {
3016 RA_IVAL_DEL(ssa_op->result_use);
3017 count--;
3018 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
3019 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain,
3020 ra,
3021 ssa, ssa_opcodes, op_array, op_array_ssa);
3022 }
3023 }
3024
3025 if (ssa_op->op1_def >= 0) {
3026 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3027 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3028 }
3029 if (ssa_op->op2_def >= 0) {
3030 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op2.var), idx);
3031 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
3032 }
3033 if (ssa_op->result_def >= 0) {
3034 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->result.var), idx);
3035 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3036 }
3037
3038 if (support_opline) {
3039 if (ssa_op->result_def >= 0
3040 && (ssa->vars[ssa_op->result_def].use_chain >= 0
3041 || ssa->vars[ssa_op->result_def].phi_use_chain)
3042 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
3043 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
3044 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
3045 || opline->opcode == ZEND_PRE_INC
3046 || opline->opcode == ZEND_PRE_DEC
3047 || opline->opcode == ZEND_POST_INC
3048 || opline->opcode == ZEND_POST_DEC
3049 || opline->opcode == ZEND_ADD
3050 || opline->opcode == ZEND_SUB
3051 || opline->opcode == ZEND_MUL
3052 || opline->opcode == ZEND_FETCH_DIM_R
3053 || opline->opcode == ZEND_FETCH_OBJ_R
3054 || opline->opcode == ZEND_FETCH_CONSTANT) {
3055 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
3056 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
3057 vars_op_array[ssa_op->result_def] = op_array;
3058 RA_IVAL_START(ssa_op->result_def, idx);
3059 count++;
3060 }
3061 }
3062 }
3063 if (ssa_op->op1_def >= 0
3064 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
3065 || ssa->vars[ssa_op->op1_def].phi_use_chain)
3066 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
3067 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)
3068 && (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
3069 || opline->opcode == ZEND_PRE_INC
3070 || opline->opcode == ZEND_PRE_DEC
3071 || opline->opcode == ZEND_POST_INC
3072 || opline->opcode == ZEND_POST_DEC)) {
3073 vars_op_array[ssa_op->op1_def] = op_array;
3074 RA_IVAL_START(ssa_op->op1_def, idx);
3075 count++;
3076 }
3077 if (ssa_op->op2_def >= 0
3078 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
3079 || ssa->vars[ssa_op->op2_def].phi_use_chain)
3080 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
3081 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)
3082 && !(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)) {
3083 vars_op_array[ssa_op->op2_def] = op_array;
3084 RA_IVAL_START(ssa_op->op2_def, idx);
3085 count++;
3086 }
3087 }
3088
3089 len = zend_jit_trace_op_len(opline);
3090 switch (opline->opcode) {
3091 case ZEND_ASSIGN_DIM:
3092 case ZEND_ASSIGN_OBJ:
3093 case ZEND_ASSIGN_STATIC_PROP:
3094 case ZEND_ASSIGN_DIM_OP:
3095 case ZEND_ASSIGN_OBJ_OP:
3096 case ZEND_ASSIGN_STATIC_PROP_OP:
3097 case ZEND_ASSIGN_OBJ_REF:
3098 case ZEND_ASSIGN_STATIC_PROP_REF:
3099 case ZEND_FRAMELESS_ICALL_3:
3100 /* OP_DATA */
3101 ssa_op++;
3102 opline++;
3103 if (ssa_op->op1_use >= 0
3104 && RA_HAS_IVAL(ssa_op->op1_use)
3105 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
3106 if (support_opline) {
3107 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain,
3108 ra,
3109 ssa, ssa_opcodes, op_array, op_array_ssa);
3110 if (opline->op1_type != IS_CV) {
3111 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_LAST_USE;
3112 }
3113 } else {
3114 RA_IVAL_DEL(ssa_op->op1_use);
3115 count--;
3116 }
3117 }
3118 if (ssa_op->op1_def >= 0) {
3119 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3120 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3121 if (support_opline
3122 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
3123 || ssa->vars[ssa_op->op1_def].phi_use_chain)
3124 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
3125 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)
3126 && !(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)) {
3127 vars_op_array[ssa_op->op1_def] = op_array;
3128 RA_IVAL_START(ssa_op->op1_def, idx);
3129 count++;
3130 }
3131 }
3132 ssa_op++;
3133 opline++;
3134 idx+=2;
3135 break;
3136 case ZEND_RECV_INIT:
3137 ssa_op++;
3138 opline++;
3139 idx++;
3140 while (opline->opcode == ZEND_RECV_INIT) {
3141 /* RECV_INIT doesn't support registers */
3142 if (ssa_op->result_use >= 0 && RA_HAS_IVAL(ssa_op->result_use)) {
3143 RA_IVAL_DEL(ssa_op->result_use);
3144 count--;
3145 }
3146 if (ssa_op->result_def >= 0) {
3147 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->result.var), idx);
3148 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3149 }
3150 ssa_op++;
3151 opline++;
3152 idx++;
3153 }
3154 break;
3155 case ZEND_BIND_GLOBAL:
3156 ssa_op++;
3157 opline++;
3158 idx++;
3159 while (opline->opcode == ZEND_BIND_GLOBAL) {
3160 /* BIND_GLOBAL doesn't support registers */
3161 if (ssa_op->op1_def >= 0) {
3162 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3163 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3164 }
3165 ssa_op++;
3166 opline++;
3167 idx++;
3168 }
3169 break;
3170 default:
3171 ssa_op += len;
3172 idx += len;
3173 break;
3174 }
3175 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3176 /* New call frames */
3177 zend_jit_trace_stack_frame *prev_frame = frame;
3178
3179 /* Clear allocated registers */
3180 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3181 j = STACK_VAR(stack, i);
3182 if (j >= 0 && RA_HAS_IVAL(j) && !(RA_IVAL_FLAGS(j) & ZREG_LAST_USE)) {
3183 RA_IVAL_DEL(j);
3184 count--;
3185 }
3186 }
3187
3188 frame = zend_jit_trace_call_frame(frame, op_array, 0);
3189 frame->prev = prev_frame;
3190 frame->func = (const zend_function*)p->op_array;
3191 stack = frame->stack;
3192 op_array = p->op_array;
3193 jit_extension =
3194 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3195 op_array_ssa = &jit_extension->func_info.ssa;
3196 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3197 for (i = 0; i < op_array->last_var; i++) {
3198 SET_STACK_VAR(stack, i, j);
3199 vars_op_array[j] = op_array;
3200 if (ssa->vars[j].use_chain >= 0
3201 && ssa->vars[j].alias == NO_ALIAS
3202 && zend_jit_var_supports_reg(ssa, j)) {
3203 RA_IVAL_START(j, idx);
3204 RA_IVAL_FLAGS(j) = ZREG_LOAD;
3205 count++;
3206 }
3207 j++;
3208 }
3209 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3210 SET_STACK_VAR(stack, i, -1);
3211 }
3212 level++;
3213 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3214 /* Close exiting call frames */
3215 for (i = 0; i < op_array->last_var; i++) {
3216 RA_IVAL_CLOSE(i, idx-1);
3217 }
3218 op_array = p->op_array;
3219 jit_extension =
3220 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3221 op_array_ssa = &jit_extension->func_info.ssa;
3222 frame = zend_jit_trace_ret_frame(frame, op_array);
3223 stack = frame->stack;
3224 if (level == 0) {
3225 /* New return frames */
3226 frame->prev = NULL;
3227 frame->func = (const zend_function*)op_array;
3228 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3229 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3230 SET_STACK_VAR(stack, i, j);
3231 vars_op_array[j] = op_array;
3232 if (ssa->vars[j].use_chain >= 0
3233 && ssa->vars[j].alias == NO_ALIAS
3234 && zend_jit_var_supports_reg(ssa, j)
3235 && !(ssa->var_info[j].type & MAY_BE_GUARD)) {
3236 RA_IVAL_START(j, idx);
3237 RA_IVAL_FLAGS(j) = ZREG_LOAD;
3238 count++;
3239 }
3240 j++;
3241 }
3242 } else {
3243 level--;
3244 }
3245 } else if (p->op == ZEND_JIT_TRACE_END) {
3246 break;
3247 }
3248 }
3249
3250 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3251 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3252 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3253 zend_ssa_phi *phi = ssa->blocks[1].phis;
3254
3255 while (phi) {
3256 i = phi->sources[1];
3257 if (RA_HAS_IVAL(i) && !ssa->vars[phi->ssa_var].no_val) {
3258 RA_IVAL_END(i, idx);
3259 RA_IVAL_FLAGS(i) &= ~ZREG_LAST_USE;
3260 }
3261 phi = phi->next;
3262 }
3263
3264 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3265 for (i = 0; i < op_array->last_var; i++) {
3266 if (RA_HAS_IVAL(i) && !ssa->vars[i].phi_use_chain) {
3267 RA_IVAL_END(i, idx);
3268 RA_IVAL_FLAGS(i) &= ~ZREG_LAST_USE;
3269 } else {
3270 RA_IVAL_CLOSE(i, idx);
3271 }
3272 }
3273 }
3274 }
3275
3276 if (count) {
3277 for (i = 0; i < ssa->vars_count; i++) {
3278 if (RA_HAS_REG(i)) {
3279 if ((RA_REG_FLAGS(i) & ZREG_LOAD) &&
3280 (RA_REG_FLAGS(i) & ZREG_LAST_USE) &&
3281 (i >= parent_vars_count || STACK_REG(parent_stack, i) == ZREG_NONE) &&
3282 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0) {
3283 /* skip life range with single use */
3284 RA_REG_DEL(i);
3285 count--;
3286 }
3287 }
3288 }
3289 }
3290
3291 if (count) {
3292 /* SSA resolution */
3293 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3294 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3295 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3296 zend_ssa_phi *phi = ssa->blocks[1].phis;
3297
3298 while (phi) {
3299 int def = phi->ssa_var;
3300 int use = phi->sources[1];
3301
3302 if (RA_HAS_REG(def)) {
3303 if (!RA_HAS_REG(use)) {
3304 RA_REG_FLAGS(def) |= ZREG_LOAD;
3305 if ((RA_REG_FLAGS(def) & ZREG_LAST_USE)
3306 && ssa->vars[def].use_chain >= 0
3307 && !ssa->vars[def].phi_use_chain
3308 && zend_ssa_next_use(ssa->ops, def, ssa->vars[def].use_chain) < 0
3309 ) {
3310 /* remove interval used once */
3311 RA_REG_DEL(def);
3312 count--;
3313 }
3314 } else if ((ssa->var_info[def].type & MAY_BE_ANY) != (ssa->var_info[use].type & MAY_BE_ANY)) {
3315 RA_REG_FLAGS(def) |= ZREG_LOAD;
3316 RA_REG_FLAGS(use) |= ZREG_STORE;
3317 } else {
3318 use = phi->sources[0];
3319 if (zend_jit_var_supports_reg(ssa, use)) {
3320 ZEND_ASSERT(!RA_HAS_REG(use));
3321 RA_REG_START(use, 0);
3322 RA_REG_FLAGS(use) = ZREG_LOAD;
3323 count++;
3324 } else {
3325 RA_REG_FLAGS(def) |= ZREG_LOAD;
3326 }
3327 }
3328 } else if (RA_HAS_REG(use)) {
3329 if (ssa->vars[use].use_chain >= 0) {
3330 RA_REG_FLAGS(use) |= ZREG_STORE; // TODO: ext/opcache/tests/jit/reg_alloc_00[67].phpt ???
3331 } else {
3332 RA_REG_DEL(use);
3333 count--;
3334 }
3335 }
3336 phi = phi->next;
3337 }
3338 } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) {
3339 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3340 int var = STACK_VAR(stack, i);
3341 if (var >= 0 && RA_HAS_REG(var)
3342 && !(RA_REG_FLAGS(var) & (ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE))) {
3343 RA_REG_FLAGS(var) |= ZREG_STORE;
3344 }
3345 }
3346 }
3347
3348 if (!count) {
3349 zend_arena_release(&CG(arena), checkpoint);
3350 return NULL;
3351 }
3352
3353 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3354 fprintf(stderr, "---- TRACE %d Live Ranges \"%s\"\n", ZEND_JIT_TRACE_NUM, op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
3355 for (i = 0; i < ssa->vars_count; i++) {
3356 if (RA_HAS_REG(i)) {
3357 fprintf(stderr, "#%d.", i);
3358 uint32_t var_num = ssa->vars[i].var;
3359 zend_dump_var(vars_op_array[i], (var_num < vars_op_array[i]->last_var ? IS_CV : 0), var_num);
3360 if (RA_REG_FLAGS(i) & ZREG_LAST_USE) {
3361 fprintf(stderr, " last_use");
3362 }
3363 if (RA_REG_FLAGS(i) & ZREG_LOAD) {
3364 fprintf(stderr, " load");
3365 }
3366 if (RA_REG_FLAGS(i) & ZREG_STORE) {
3367 fprintf(stderr, " store");
3368 }
3369 fprintf(stderr, "\n");
3370 }
3371 }
3372 fprintf(stderr, "\n");
3373 }
3374
3375 return ra;
3376 }
3377
3378 zend_arena_release(&CG(arena), checkpoint);
3379 return NULL;
3380 }
3381
zend_jit_trace_cleanup_stack(zend_jit_ctx * jit,zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes)3382 static void zend_jit_trace_cleanup_stack(zend_jit_ctx *jit, zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes)
3383 {
3384 if (ssa_op->op1_use >= 0
3385 && jit->ra[ssa_op->op1_use].ref
3386 && (jit->ra[ssa_op->op1_use].flags & ZREG_LAST_USE)
3387 && (ssa_op->op1_use_chain == -1
3388 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->op1_use_chain], ssa->ops + ssa_op->op1_use_chain, ssa_op->op1_use))) {
3389 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var));
3390 }
3391 if (ssa_op->op2_use >= 0
3392 && ssa_op->op2_use != ssa_op->op1_use
3393 && jit->ra[ssa_op->op2_use].ref
3394 && (jit->ra[ssa_op->op2_use].flags & ZREG_LAST_USE)
3395 && (ssa_op->op2_use_chain == -1
3396 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->op2_use_chain], ssa->ops + ssa_op->op2_use_chain, ssa_op->op2_use))) {
3397 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->op2.var));
3398 }
3399 if (ssa_op->result_use >= 0
3400 && ssa_op->result_use != ssa_op->op1_use
3401 && ssa_op->result_use != ssa_op->op2_use
3402 && jit->ra[ssa_op->result_use].ref
3403 && (jit->ra[ssa_op->result_use].flags & ZREG_LAST_USE)
3404 && (ssa_op->res_use_chain == -1
3405 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->res_use_chain], ssa->ops + ssa_op->res_use_chain, ssa_op->result_use))) {
3406 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
3407 }
3408 }
3409
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3410 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3411 {
3412 zend_op *next_opline = (zend_op*)(opline + 1);
3413
3414 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3415 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3416 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3417 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3418 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3419 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3420 }
3421 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3422 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3423 }
3424 }
3425
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3426 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3427 {
3428 int var = ssa_op->result_def;
3429 int i;
3430 int use = ssa->vars[var].use_chain;
3431 const zend_op *opline;
3432
3433 if (use < 0
3434 || ssa->vars[var].phi_use_chain
3435 || ssa->ops[use].op1_use != var
3436 || ssa->ops[use].op1_use_chain != -1) {
3437 return 0;
3438 }
3439
3440 opline = ssa_opcodes[use];
3441 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3442 return (opline->op2_type == IS_CONST &&
3443 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3444 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3445 if (!JIT_G(current_frame)
3446 || !JIT_G(current_frame)->call
3447 || !JIT_G(current_frame)->call->func
3448 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3449 return 0;
3450 }
3451 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3452 && opline->opcode != ZEND_FETCH_OBJ_IS
3453 && opline->opcode != ZEND_FETCH_OBJ_W
3454 && opline->opcode != ZEND_ASSIGN_OBJ
3455 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3456 && opline->opcode != ZEND_PRE_INC_OBJ
3457 && opline->opcode != ZEND_PRE_DEC_OBJ
3458 && opline->opcode != ZEND_POST_INC_OBJ
3459 && opline->opcode != ZEND_POST_DEC_OBJ) {
3460 return 0;
3461 }
3462
3463 if (opline->op2_type != IS_CONST
3464 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3465 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3466 return 0;
3467 }
3468
3469 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3470 if (opline->op1_type == IS_CV
3471 && (opline+1)->op1_type == IS_CV
3472 && (opline+1)->op1.var == opline->op1.var) {
3473 /* skip $a->prop += $a; */
3474 return 0;
3475 }
3476 if (!zend_jit_supported_binary_op(
3477 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3478 return 0;
3479 }
3480 }
3481
3482 for (i = ssa->vars[var].definition; i < use; i++) {
3483 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3484 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3485 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3486 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3487 return 0;
3488 }
3489 }
3490
3491 return 1;
3492 }
3493
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3494 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3495 {
3496 uint32_t i;
3497
3498 for (i = 0; i < stack_size; i++) {
3499 if (STACK_FLAGS(stack, i) & ~(ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE)) {
3500 return 1;
3501 } else if (STACK_REG(stack, i) != ZREG_NONE) {
3502 return 1;
3503 }
3504 }
3505 return 0;
3506 }
3507
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3508 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3509 {
3510 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3511 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3512 uint32_t stack_size;
3513 zend_jit_trace_stack *stack;
3514
3515 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3516 return 1;
3517 }
3518
3519 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3520 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3521 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3522 }
3523
zend_jit_trace_deoptimization(zend_jit_ctx * jit,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_jit_exit_const * constants,int8_t func_reg,bool polymorphic_side_trace)3524 static int zend_jit_trace_deoptimization(
3525 zend_jit_ctx *jit,
3526 uint32_t flags,
3527 const zend_op *opline,
3528 zend_jit_trace_stack *parent_stack,
3529 int parent_vars_count,
3530 zend_ssa *ssa,
3531 zend_jit_trace_stack *stack,
3532 zend_jit_exit_const *constants,
3533 int8_t func_reg,
3534 bool polymorphic_side_trace)
3535 {
3536 int i;
3537 int check2 = -1;
3538
3539 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3540 for (i = 0; i < parent_vars_count; i++) {
3541 int8_t reg = STACK_REG(parent_stack, i);
3542
3543 if (STACK_FLAGS(parent_stack, i) == ZREG_CONST) {
3544 uint8_t type = STACK_TYPE(parent_stack, i);
3545
3546 if (type == IS_LONG) {
3547 if (!zend_jit_store_const_long(jit, i,
3548 (zend_long)constants[STACK_REF(parent_stack, i)].i)) {
3549 return 0;
3550 }
3551 } else if (type == IS_DOUBLE) {
3552 if (!zend_jit_store_const_double(jit, i,
3553 constants[STACK_REF(parent_stack, i)].d)) {
3554 return 0;
3555 }
3556 } else {
3557 ZEND_UNREACHABLE();
3558 }
3559 if (stack) {
3560 SET_STACK_TYPE(stack, i, type, 1);
3561 if (jit->ra && jit->ra[i].ref) {
3562 SET_STACK_REF(stack, i, jit->ra[i].ref);
3563 }
3564 }
3565 } else if (STACK_FLAGS(parent_stack, i) == ZREG_TYPE_ONLY) {
3566 uint8_t type = STACK_TYPE(parent_stack, i);
3567
3568 if (!zend_jit_store_type(jit, i, type)) {
3569 return 0;
3570 }
3571 if (stack) {
3572 SET_STACK_TYPE(stack, i, type, 1);
3573 }
3574 } else if (STACK_FLAGS(parent_stack, i) == ZREG_THIS) {
3575 if (polymorphic_side_trace) {
3576 ssa->var_info[i].delayed_fetch_this = 1;
3577 if (stack) {
3578 SET_STACK_REG_EX(stack, i, ZREG_NONE, ZREG_THIS);
3579 }
3580 } else if (!zend_jit_load_this(jit, EX_NUM_TO_VAR(i))) {
3581 return 0;
3582 }
3583 } else if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_ADDREF) {
3584 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(i));
3585 zend_jit_zval_try_addref(jit, dst);
3586 } else if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
3587 ZEND_ASSERT(reg != ZREG_NONE);
3588 ZEND_ASSERT(check2 == -1);
3589 check2 = i;
3590 } else if (STACK_FLAGS(parent_stack, i) & ZREG_SPILL_SLOT) {
3591 if (ssa && ssa->vars[i].no_val) {
3592 /* pass */
3593 } else {
3594 uint8_t type = STACK_TYPE(parent_stack, i);
3595
3596 if (!zend_jit_store_spill_slot(jit, 1 << type, i, reg, STACK_REF(parent_stack, i),
3597 STACK_MEM_TYPE(parent_stack, i) != type)) {
3598 return 0;
3599 }
3600 if (stack) {
3601 if (jit->ra && jit->ra[i].ref) {
3602 SET_STACK_TYPE(stack, i, type, 0);
3603 if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
3604 SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
3605 } else {
3606 SET_STACK_REF(stack, i, jit->ra[i].ref);
3607 }
3608 } else {
3609 SET_STACK_TYPE(stack, i, type, 1);
3610 }
3611 }
3612 }
3613 } else if (reg != ZREG_NONE) {
3614 if (ssa && ssa->vars[i].no_val) {
3615 /* pass */
3616 } else {
3617 uint8_t type = STACK_TYPE(parent_stack, i);
3618
3619 if (!zend_jit_store_reg(jit, 1 << type, i, reg,
3620 (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0,
3621 STACK_MEM_TYPE(parent_stack, i) != type)) {
3622 return 0;
3623 }
3624 if (stack) {
3625 if (jit->ra && jit->ra[i].ref) {
3626 SET_STACK_TYPE(stack, i, type, 0);
3627 if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
3628 SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
3629 } else {
3630 SET_STACK_REF(stack, i, jit->ra[i].ref);
3631 }
3632 } else {
3633 SET_STACK_TYPE(stack, i, type, 1);
3634 }
3635 }
3636 }
3637 }
3638 }
3639
3640 if (check2 != -1) {
3641 int8_t reg = STACK_REG(parent_stack, check2);
3642
3643 ZEND_ASSERT(STACK_FLAGS(parent_stack, check2) == ZREG_ZVAL_COPY);
3644 ZEND_ASSERT(reg != ZREG_NONE);
3645 if (!zend_jit_escape_if_undef(jit, check2, flags, opline, reg)) {
3646 return 0;
3647 }
3648 if (!zend_jit_restore_zval(jit, EX_NUM_TO_VAR(check2), reg)) {
3649 return 0;
3650 }
3651 }
3652
3653 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3654 if (!zend_jit_save_call_chain(jit, -1)) {
3655 return 0;
3656 }
3657 }
3658
3659 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3660 const zend_op *op = opline - 1;
3661
3662 if (!zend_jit_free_op(jit, op, -1, op->op2.var)) {
3663 return 0;
3664 }
3665 }
3666
3667 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3668 const zend_op *op = opline - 1;
3669
3670 if (!zend_jit_free_op(jit, op, -1, op->op1.var)) {
3671 return 0;
3672 }
3673 }
3674
3675 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3676 zend_jit_check_exception(jit);
3677 }
3678
3679 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3680 if (!zend_jit_free_trampoline(jit, func_reg)) {
3681 return 0;
3682 }
3683 }
3684
3685 return 1;
3686 }
3687
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3688 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3689 {
3690 info->has_range = 1;
3691 info->range.min = min;
3692 info->range.max = max;
3693 info->range.underflow = 0;
3694 info->range.overflow = 0;
3695 }
3696
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3697 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3698 {
3699 zend_long op1_min, op1_max, op2_min, op2_max;
3700
3701 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3702 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3703 return;
3704 }
3705
3706 op1_min = OP1_MIN_RANGE();
3707 op1_max = OP1_MAX_RANGE();
3708 op2_min = OP2_MIN_RANGE();
3709 op2_max = OP2_MAX_RANGE();
3710
3711 switch (opline->opcode) {
3712 case ZEND_IS_EQUAL:
3713 case ZEND_CASE:
3714 case ZEND_IS_IDENTICAL:
3715 case ZEND_CASE_STRICT:
3716 case ZEND_IS_NOT_IDENTICAL:
3717 if (!exit_if_true) {
3718 /* op1 == op2 */
3719 if (ssa_op->op1_use >= 0) {
3720 zend_jit_trace_set_var_range(
3721 &ssa->var_info[ssa_op->op1_use],
3722 MAX(op1_min, op2_min),
3723 MIN(op1_max, op2_max));
3724 }
3725 if (ssa_op->op2_use >= 0) {
3726 zend_jit_trace_set_var_range(
3727 &ssa->var_info[ssa_op->op2_use],
3728 MAX(op2_min, op1_min),
3729 MIN(op2_max, op1_max));
3730 }
3731 }
3732 break;
3733 case ZEND_IS_NOT_EQUAL:
3734 if (exit_if_true) {
3735 /* op1 == op2 */
3736 if (ssa_op->op1_use >= 0) {
3737 zend_jit_trace_set_var_range(
3738 &ssa->var_info[ssa_op->op1_use],
3739 MAX(op1_min, op2_min),
3740 MIN(op1_max, op2_max));
3741 }
3742 if (ssa_op->op2_use >= 0) {
3743 zend_jit_trace_set_var_range(
3744 &ssa->var_info[ssa_op->op2_use],
3745 MAX(op2_min, op1_min),
3746 MIN(op2_max, op1_max));
3747 }
3748 }
3749 break;
3750 case ZEND_IS_SMALLER_OR_EQUAL:
3751 if (!exit_if_true) {
3752 /* op1 <= op2 */
3753 if (ssa_op->op1_use >= 0) {
3754 zend_jit_trace_set_var_range(
3755 &ssa->var_info[ssa_op->op1_use],
3756 op1_min,
3757 MIN(op1_max, op2_max));
3758 }
3759 if (ssa_op->op2_use >= 0) {
3760 zend_jit_trace_set_var_range(
3761 &ssa->var_info[ssa_op->op2_use],
3762 MAX(op2_min, op1_min),
3763 op2_max);
3764 }
3765 } else {
3766 /* op1 > op2 */
3767 if (ssa_op->op1_use >= 0) {
3768 zend_jit_trace_set_var_range(
3769 &ssa->var_info[ssa_op->op1_use],
3770 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3771 op1_max);
3772 }
3773 if (ssa_op->op2_use >= 0) {
3774 zend_jit_trace_set_var_range(
3775 &ssa->var_info[ssa_op->op2_use],
3776 op2_min,
3777 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3778 }
3779 }
3780 break;
3781 case ZEND_IS_SMALLER:
3782 if (!exit_if_true) {
3783 /* op1 < op2 */
3784 if (ssa_op->op1_use >= 0) {
3785 zend_jit_trace_set_var_range(
3786 &ssa->var_info[ssa_op->op1_use],
3787 op1_min,
3788 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3789 }
3790 if (ssa_op->op2_use >= 0) {
3791 zend_jit_trace_set_var_range(
3792 &ssa->var_info[ssa_op->op2_use],
3793 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3794 op2_max);
3795 }
3796 } else {
3797 /* op1 >= op2 */
3798 if (ssa_op->op1_use >= 0) {
3799 zend_jit_trace_set_var_range(
3800 &ssa->var_info[ssa_op->op1_use],
3801 MAX(op1_min, op2_min),
3802 op1_max);
3803 }
3804 if (ssa_op->op2_use >= 0) {
3805 zend_jit_trace_set_var_range(
3806 &ssa->var_info[ssa_op->op2_use],
3807 op2_min,
3808 MIN(op2_max, op1_max));
3809 }
3810 }
3811 break;
3812 }
3813 }
3814
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3815 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3816 {
3817 uint8_t prev_opcode;
3818
3819 if (opline->op1_type == IS_CONST
3820 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3821 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3822 if (ssa_op->op2_use >= 0) {
3823 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3824 ssa_op--;
3825 opline = ssa_opcodes[ssa_op - ssa->ops];
3826 prev_opcode = opline->opcode;
3827 if (prev_opcode == ZEND_PRE_INC
3828 || prev_opcode == ZEND_PRE_DEC
3829 || prev_opcode == ZEND_POST_INC
3830 || prev_opcode == ZEND_POST_DEC) {
3831 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3832 }
3833 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3834 ssa_op--;
3835 opline = ssa_opcodes[ssa_op - ssa->ops];
3836 prev_opcode = opline->opcode;
3837 if (prev_opcode == ZEND_ADD
3838 || prev_opcode == ZEND_SUB) {
3839 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3840 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3841 }
3842 }
3843 }
3844 } else if (opline->op2_type == IS_CONST
3845 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3846 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3847 if (ssa_op->op1_use >= 0) {
3848 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3849 ssa_op--;
3850 opline = ssa_opcodes[ssa_op - ssa->ops];
3851 prev_opcode = opline->opcode;
3852 if (prev_opcode == ZEND_PRE_INC
3853 || prev_opcode == ZEND_PRE_DEC
3854 || prev_opcode == ZEND_POST_INC
3855 || prev_opcode == ZEND_POST_DEC) {
3856 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3857 }
3858 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3859 ssa_op--;
3860 opline = ssa_opcodes[ssa_op - ssa->ops];
3861 prev_opcode = opline->opcode;
3862 if (prev_opcode == ZEND_ADD
3863 || prev_opcode == ZEND_SUB) {
3864 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3865 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3866 }
3867 }
3868 }
3869 } else {
3870 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3871 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3872
3873 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3874 && prev_ssa_op != ssa->ops
3875 && prev_ssa_op->op1_use >= 0
3876 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3877 prev_ssa_op--;
3878 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3879 }
3880
3881 if (ssa_op->op1_use == prev_ssa_op->op1_use
3882 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3883 if (prev_opcode == ZEND_IS_EQUAL
3884 || prev_opcode == ZEND_IS_NOT_EQUAL
3885 || prev_opcode == ZEND_IS_SMALLER
3886 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3887 || prev_opcode == ZEND_CASE
3888 || prev_opcode == ZEND_IS_IDENTICAL
3889 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3890 || prev_opcode == ZEND_CASE_STRICT) {
3891 if (ssa_op->op1_use < 0) {
3892 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3893 return 0;
3894 }
3895 }
3896 if (ssa_op->op2_use < 0) {
3897 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3898 return 0;
3899 }
3900 }
3901 return 1;
3902 }
3903 }
3904 }
3905 return 0;
3906 }
3907
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3908 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3909 zend_jit_trace_rec *p,
3910 zend_jit_trace_stack_frame *frame)
3911 {
3912 if (opline->result_type == IS_TMP_VAR
3913 && (p+1)->op == ZEND_JIT_TRACE_VM
3914 && (p+1)->opline == opline + 1
3915 && ((opline+1)->opcode == ZEND_SEND_VAL
3916 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3917 && frame
3918 && frame->call
3919 && frame->call->func
3920 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3921 && (opline+1)->op1_type == IS_TMP_VAR
3922 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3923 && (opline+1)->op1.var == opline->result.var) {
3924
3925 if (frame->call && frame->call->func) {
3926 uint8_t res_type = (p+1)->op1_type;
3927
3928 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3929 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3930 }
3931 }
3932 return 1;
3933 }
3934 return 0;
3935 }
3936
zend_jit_find_ssa_var(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t opline_num,uint32_t var_num)3937 static int zend_jit_find_ssa_var(const zend_op_array *op_array,
3938 const zend_ssa *ssa,
3939 uint32_t opline_num,
3940 uint32_t var_num)
3941 {
3942 int ssa_var, j, b = ssa->cfg.map[opline_num];
3943 const zend_basic_block *bb = ssa->cfg.blocks + b;
3944 const zend_ssa_phi *phi;
3945 const zend_ssa_op *ssa_op;
3946 zend_worklist worklist;
3947 ALLOCA_FLAG(use_heap)
3948
3949 while (1) {
3950 ssa_op = ssa->ops + opline_num;
3951 ssa_var = ssa_op->result_def;
3952 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3953 return ssa_var;
3954 }
3955 ssa_var = ssa_op->op2_def;
3956 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3957 return ssa_var;
3958 }
3959 ssa_var = ssa_op->op1_def;
3960 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3961 return ssa_var;
3962 }
3963 if (opline_num == bb->start) {
3964 break;
3965 }
3966 opline_num--;
3967 }
3968 phi = ssa->blocks[b].phis;
3969 ssa_var = -1;
3970 while (phi) {
3971 if (phi->var == var_num) {
3972 ssa_var = phi->ssa_var;
3973 }
3974 phi = phi->next;
3975 }
3976 if (ssa_var >= 0) {
3977 return ssa_var;
3978 }
3979
3980 if (!bb->predecessors_count) {
3981 return -1;
3982 }
3983
3984 ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
3985
3986 for (j = 0; j < bb->predecessors_count; j++) {
3987 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
3988 zend_worklist_push(&worklist, b);
3989 }
3990
3991 while (zend_worklist_len(&worklist) != 0) {
3992 b = zend_worklist_pop(&worklist);
3993 bb = &ssa->cfg.blocks[b];
3994 if (bb->len) {
3995 opline_num = bb->start + bb->len - 1;
3996 while (1) {
3997 ssa_op = ssa->ops + opline_num;
3998 ssa_var = ssa_op->result_def;
3999 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
4000 goto found;
4001 }
4002 ssa_var = ssa_op->op2_def;
4003 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
4004 goto found;
4005 }
4006 ssa_var = ssa_op->op1_def;
4007 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
4008 goto found;
4009 }
4010 if (opline_num == bb->start) {
4011 break;
4012 }
4013 opline_num--;
4014 }
4015 }
4016 phi = ssa->blocks[b].phis;
4017 ssa_var = -1;
4018 while (phi) {
4019 if (phi->var == var_num) {
4020 ssa_var = phi->ssa_var;
4021 }
4022 phi = phi->next;
4023 }
4024 if (ssa_var >= 0) {
4025 goto found;
4026 }
4027 for (j = 0; j < bb->predecessors_count; j++) {
4028 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
4029 zend_worklist_push(&worklist, b);
4030 }
4031 }
4032 ssa_var = -1;
4033
4034 found:
4035 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
4036 return ssa_var;
4037 }
4038
zend_jit_trace_must_store_type(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t opline_num,uint32_t var_num,uint8_t type)4039 static bool zend_jit_trace_must_store_type(const zend_op_array *op_array,
4040 const zend_ssa *ssa,
4041 uint32_t opline_num,
4042 uint32_t var_num,
4043 uint8_t type)
4044 {
4045 if (ssa->var_info) {
4046 int ssa_var = zend_jit_find_ssa_var(op_array, ssa, opline_num, var_num);
4047
4048 if (ssa_var >= 0) {
4049 if ((ssa->var_info[ssa_var].type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1U << type)) {
4050 return 0;
4051 }
4052 }
4053 }
4054 return 1;
4055 }
4056
zend_jit_trace_may_throw(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,const zend_ssa * ssa,uint32_t t1,uint32_t t2,uint32_t t3,uint32_t val_type)4057 static bool zend_jit_trace_may_throw(const zend_op *opline,
4058 const zend_ssa_op *ssa_op,
4059 const zend_op_array *op_array,
4060 const zend_ssa *ssa,
4061 uint32_t t1,
4062 uint32_t t2,
4063 uint32_t t3,
4064 uint32_t val_type)
4065 {
4066 switch (opline->opcode) {
4067 case ZEND_ASSIGN_DIM_OP:
4068 if (opline->extended_value != ZEND_CONCAT
4069 && val_type == IS_LONG
4070 && (t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_ARRAY
4071 && MAY_BE_PACKED_ONLY(t1)
4072 && !(t1 & MAY_BE_ARRAY_OF_REF)
4073 && (t2 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_LONG
4074 && (t3 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_LONG) {
4075 return 0;
4076 }
4077 break;
4078 default:
4079 break;
4080 }
4081 return zend_may_throw_ex(opline, ssa_op, op_array, ssa, t1, t2);
4082 }
4083
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)4084 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
4085 {
4086 const void *handler = NULL;
4087 zend_jit_ctx ctx;
4088 zend_jit_ctx *jit = &ctx;
4089 zend_jit_reg_var *ra = NULL;
4090 zend_script *script = NULL;
4091 zend_string *name = NULL;
4092 void *checkpoint;
4093 const zend_op_array *op_array;
4094 zend_ssa *ssa, *op_array_ssa;
4095 const zend_op **ssa_opcodes;
4096 zend_jit_trace_rec *p;
4097 zend_jit_op_array_trace_extension *jit_extension;
4098 int num_op_arrays = 0;
4099 zend_jit_trace_info *t;
4100 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
4101 uint8_t smart_branch_opcode;
4102 const void *exit_addr;
4103 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info, op1_mem_info;
4104 bool send_result = 0;
4105 bool skip_comparison;
4106 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
4107 zend_class_entry *ce;
4108 bool ce_is_instanceof;
4109 bool on_this = 0;
4110 bool delayed_fetch_this = 0;
4111 bool avoid_refcounting = 0;
4112 bool polymorphic_side_trace =
4113 parent_trace &&
4114 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
4115 uint32_t i;
4116 zend_jit_trace_stack_frame *frame, *top, *call;
4117 zend_jit_trace_stack *stack;
4118 uint8_t res_type = IS_UNKNOWN;
4119 const zend_op *opline, *orig_opline;
4120 const zend_ssa_op *ssa_op, *orig_ssa_op;
4121 int checked_stack;
4122 int peek_checked_stack;
4123 uint32_t frame_flags = 0;
4124
4125 JIT_G(current_trace) = trace_buffer;
4126
4127 checkpoint = zend_arena_checkpoint(CG(arena));
4128
4129 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4130
4131 if (!ssa) {
4132 goto jit_cleanup;
4133 }
4134
4135 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4136
4137 op_array = trace_buffer->op_array;
4138 opline = trace_buffer[1].opline;
4139 name = zend_jit_trace_name(op_array, opline->lineno);
4140 zend_jit_trace_start(&ctx, op_array, ssa, name, ZEND_JIT_TRACE_NUM,
4141 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4142 ctx.trace = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
4143
4144 /* Register allocation */
4145 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4146 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4147 ctx.ra = ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4148 }
4149
4150 p = trace_buffer;
4151 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4152 op_array = p->op_array;
4153 frame = JIT_G(current_frame);
4154 top = zend_jit_trace_call_frame(frame, op_array, 0);
4155 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4156 frame->used_stack = checked_stack = peek_checked_stack = 0;
4157 stack = frame->stack;
4158 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4159 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4160 }
4161
4162 opline = p[1].opline;
4163 p += ZEND_JIT_TRACE_START_REC_SIZE;
4164
4165 jit_extension =
4166 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4167 op_array_ssa = &jit_extension->func_info.ssa;
4168
4169 if (!parent_trace) {
4170 zend_jit_set_last_valid_opline(&ctx, opline);
4171 zend_jit_track_last_valid_opline(&ctx);
4172 } else {
4173 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4174 zend_jit_trace_opline_guard(&ctx, opline);
4175 } else {
4176 zend_jit_reset_last_valid_opline(&ctx);
4177 }
4178 }
4179
4180 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4181 int last_var;
4182 int parent_vars_count = 0;
4183 zend_jit_trace_stack *parent_stack = NULL;
4184 int used_stack = ((zend_tssa*)ssa)->used_stack;
4185
4186 if (used_stack > 0) {
4187 peek_checked_stack = used_stack;
4188 if (!zend_jit_stack_check(&ctx, opline, used_stack)) {
4189 goto jit_failure;
4190 }
4191 }
4192
4193 if (parent_trace) {
4194 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4195 op_array->last_var + op_array->T);
4196 if (parent_vars_count) {
4197 parent_stack =
4198 zend_jit_traces[parent_trace].stack_map +
4199 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4200 }
4201 }
4202
4203 last_var = op_array->last_var;
4204 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4205 last_var += op_array->T;
4206 }
4207
4208 for (i = 0; i < last_var; i++) {
4209 uint32_t info = ssa->var_info[i].type;
4210
4211 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4212 uint8_t type, mem_type;
4213
4214 type = concrete_type(info);
4215 if (i < parent_vars_count
4216 && STACK_TYPE(parent_stack, i) == type) {
4217 mem_type = STACK_MEM_TYPE(parent_stack, i);
4218 if (mem_type != IS_UNKNOWN) {
4219 SET_STACK_TYPE(stack, i, mem_type, 1);
4220 }
4221 SET_STACK_TYPE(stack, i, type, 0);
4222 } else {
4223 SET_STACK_TYPE(stack, i, type, 1);
4224 }
4225 } else if (ssa->vars[i].alias != NO_ALIAS) {
4226 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4227 } else if (i < parent_vars_count
4228 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4229 /* This must be already handled by trace type inference */
4230 ZEND_ASSERT(ssa->vars[i].use_chain < 0 && !ssa->vars[i].phi_use_chain);
4231 SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i), 1);
4232 } else if ((info & MAY_BE_GUARD) != 0
4233 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4234 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4235 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4236 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4237 && (ssa->vars[i].use_chain != -1
4238 || (ssa->vars[i].phi_use_chain
4239 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4240 /* Check loop-invariant variable type */
4241 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4242 goto jit_failure;
4243 }
4244 info &= ~MAY_BE_GUARD;
4245 ssa->var_info[i].type = info;
4246 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4247 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4248 && op_array->function_name
4249 && i >= op_array->num_args) {
4250 /* This must be already handled by trace type inference */
4251 ZEND_UNREACHABLE();
4252 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4253 }
4254
4255 if ((info & MAY_BE_PACKED_GUARD) != 0
4256 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4257 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4258 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4259 && (ssa->vars[i].use_chain != -1
4260 || (ssa->vars[i].phi_use_chain
4261 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4262 if (!zend_jit_packed_guard(&ctx, opline, EX_NUM_TO_VAR(i), info)) {
4263 goto jit_failure;
4264 }
4265 info &= ~MAY_BE_PACKED_GUARD;
4266 ssa->var_info[i].type = info;
4267 }
4268 }
4269
4270 if (parent_trace) {
4271 /* Deoptimization */
4272 if (!zend_jit_trace_deoptimization(&ctx,
4273 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4274 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4275 parent_stack, parent_vars_count, ssa, stack,
4276 zend_jit_traces[parent_trace].constants,
4277 zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg,
4278 polymorphic_side_trace)) {
4279 goto jit_failure;
4280 }
4281 }
4282
4283 if (ra
4284 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4285 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4286 for (i = 0; i < last_var; i++) {
4287 if (RA_HAS_REG(i)
4288 && (RA_REG_FLAGS(i) & ZREG_LOAD) != 0
4289 && ra[i].ref != STACK_REF(stack, i)
4290 ) {
4291
4292 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4293 uint8_t op_type;
4294
4295 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4296 op_type = concrete_type(ssa->var_info[i].type);
4297 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), op_type)) {
4298 goto jit_failure;
4299 }
4300 SET_STACK_TYPE(stack, i, op_type, 1);
4301 }
4302
4303 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
4304 if (!zend_jit_load_var(&ctx, ssa->var_info[i].type, i, i)) {
4305 goto jit_failure;
4306 }
4307 SET_STACK_REF_EX(stack, i, ra[i].ref, ZREG_LOAD);
4308 } else {
4309 SET_STACK_REF_EX(stack, i, IR_NULL, ZREG_LOAD);
4310 }
4311 }
4312 }
4313 }
4314 }
4315
4316 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4317 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4318 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4319
4320 jit->trace_loop_ref = zend_jit_trace_begin_loop(&ctx); /* start of of trace loop */
4321
4322 if (ra) {
4323 zend_ssa_phi *phi = ssa->blocks[1].phis;
4324
4325 /* First try to insert IR Phi */
4326 while (phi) {
4327 if (RA_HAS_REG(phi->ssa_var)
4328 && !(RA_REG_FLAGS(phi->ssa_var) & ZREG_LOAD)) {
4329 zend_jit_trace_gen_phi(&ctx, phi);
4330 SET_STACK_REF(stack, phi->var, ra[phi->ssa_var].ref);
4331 }
4332 phi = phi->next;
4333 }
4334
4335 phi = ssa->blocks[1].phis;
4336 while (phi) {
4337 if (RA_HAS_REG(phi->ssa_var)) {
4338 if (RA_REG_FLAGS(phi->ssa_var) & ZREG_LOAD) {
4339 uint32_t info = ssa->var_info[phi->ssa_var].type;
4340
4341 if (info & MAY_BE_GUARD) {
4342 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4343 goto jit_failure;
4344 }
4345 info &= ~MAY_BE_GUARD;
4346 ssa->var_info[phi->ssa_var].type = info;
4347 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4348 }
4349 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
4350 goto jit_failure;
4351 }
4352 SET_STACK_REF_EX(stack, phi->var, ra[phi->ssa_var].ref, ZREG_LOAD);
4353 } else if (RA_REG_FLAGS(phi->ssa_var) & ZREG_STORE) {
4354
4355 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var,
4356 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4357 goto jit_failure;
4358 }
4359 SET_STACK_REF_EX(stack, phi->var, ra[phi->ssa_var].ref, ZREG_STORE);
4360 } else {
4361 /* Register has to be written back on side exit */
4362 SET_STACK_REF(stack, phi->var, ra[phi->ssa_var].ref);
4363 }
4364 }
4365 phi = phi->next;
4366 }
4367 }
4368
4369 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4370 // if (ra && dzend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4371 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4372 //
4373 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4374 // if (!timeout_exit_addr) {
4375 // goto jit_failure;
4376 // }
4377 // }
4378 // }
4379
4380 }
4381
4382 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4383 for (;;p++) {
4384 if (p->op == ZEND_JIT_TRACE_VM) {
4385 uint8_t op1_type = p->op1_type;
4386 uint8_t op2_type = p->op2_type;
4387 uint8_t op3_type = p->op3_type;
4388 uint8_t orig_op1_type = op1_type;
4389 uint8_t orig_op2_type = op2_type;
4390 uint8_t val_type = IS_UNKNOWN;
4391 bool op1_indirect;
4392 zend_class_entry *op1_ce = NULL;
4393 zend_class_entry *op2_ce = NULL;
4394 bool gen_handler = false;
4395
4396 opline = p->opline;
4397 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4398 op1_type = IS_UNKNOWN;
4399 }
4400 if (op1_type != IS_UNKNOWN) {
4401 op1_type &= ~IS_TRACE_PACKED;
4402 }
4403 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4404 op2_type = IS_UNKNOWN;
4405 }
4406 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4407 op3_type = IS_UNKNOWN;
4408 }
4409
4410 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4411 op1_ce = (zend_class_entry*)(p+1)->ce;
4412 p++;
4413 }
4414 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4415 op2_ce = (zend_class_entry*)(p+1)->ce;
4416 p++;
4417 }
4418 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4419 val_type = (p+1)->op1_type;
4420 p++;
4421 }
4422
4423 frame_flags = 0;
4424
4425 switch (opline->opcode) {
4426 case ZEND_INIT_FCALL:
4427 case ZEND_INIT_FCALL_BY_NAME:
4428 case ZEND_INIT_NS_FCALL_BY_NAME:
4429 case ZEND_INIT_METHOD_CALL:
4430 case ZEND_INIT_DYNAMIC_CALL:
4431 case ZEND_INIT_STATIC_METHOD_CALL:
4432 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
4433 case ZEND_INIT_USER_CALL:
4434 case ZEND_NEW:
4435 frame->call_level++;
4436 }
4437
4438 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4439 switch (opline->opcode) {
4440 case ZEND_PRE_INC:
4441 case ZEND_PRE_DEC:
4442 case ZEND_POST_INC:
4443 case ZEND_POST_DEC:
4444 if (opline->op1_type != IS_CV) {
4445 break;
4446 }
4447 op1_info = OP1_INFO();
4448 CHECK_OP1_TRACE_TYPE();
4449 if (!(op1_info & MAY_BE_LONG)) {
4450 break;
4451 }
4452 if (opline->result_type != IS_UNUSED) {
4453 res_use_info = zend_jit_trace_type_to_info(
4454 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4455 if (opline->result_type == IS_CV) {
4456 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4457 }
4458 res_info = RES_INFO();
4459 res_addr = RES_REG_ADDR();
4460 } else {
4461 res_use_info = -1;
4462 res_info = -1;
4463 res_addr = 0;
4464 }
4465 op1_def_info = OP1_DEF_INFO();
4466 if (op1_def_info & MAY_BE_GUARD
4467 && !has_concrete_type(op1_def_info)) {
4468 op1_def_info &= ~MAY_BE_GUARD;
4469 }
4470 if (!zend_jit_inc_dec(&ctx, opline,
4471 op1_info, OP1_REG_ADDR(),
4472 op1_def_info, OP1_DEF_REG_ADDR(),
4473 res_use_info, res_info,
4474 res_addr,
4475 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4476 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4477 goto jit_failure;
4478 }
4479 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4480 && !(op1_info & MAY_BE_STRING)) {
4481 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4482 if (opline->result_type != IS_UNUSED) {
4483 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4484 }
4485 } else if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)
4486 && !(op1_info & MAY_BE_STRING)) {
4487 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4488 if (opline->result_type != IS_UNUSED) {
4489 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4490 }
4491 }
4492 if (opline->result_type != IS_UNUSED
4493 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4494 && !(op1_info & MAY_BE_STRING)) {
4495 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4496 } else if (opline->result_type != IS_UNUSED
4497 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)
4498 && !(res_info & MAY_BE_STRING)) {
4499 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4500 }
4501 goto done;
4502 case ZEND_BW_OR:
4503 case ZEND_BW_AND:
4504 case ZEND_BW_XOR:
4505 case ZEND_SL:
4506 case ZEND_SR:
4507 case ZEND_MOD:
4508 op1_info = OP1_INFO();
4509 CHECK_OP1_TRACE_TYPE();
4510 op2_info = OP2_INFO();
4511 CHECK_OP2_TRACE_TYPE();
4512 if (!(op1_info & MAY_BE_LONG)
4513 || !(op2_info & MAY_BE_LONG)) {
4514 break;
4515 }
4516 res_addr = RES_REG_ADDR();
4517 if (Z_MODE(res_addr) != IS_REG
4518 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4519 send_result = 1;
4520 res_use_info = -1;
4521 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4522 if (!zend_jit_reuse_ip(&ctx)) {
4523 goto jit_failure;
4524 }
4525 } else {
4526 res_use_info = zend_jit_trace_type_to_info(
4527 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4528 if (opline->result_type == IS_CV) {
4529 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4530 }
4531 }
4532 res_info = RES_INFO();
4533 if (!zend_jit_long_math(&ctx, opline,
4534 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4535 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4536 res_use_info, res_info, res_addr,
4537 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4538 goto jit_failure;
4539 }
4540 goto done;
4541 case ZEND_ADD:
4542 case ZEND_SUB:
4543 case ZEND_MUL:
4544 // case ZEND_DIV: // TODO: check for division by zero ???
4545 op1_info = OP1_INFO();
4546 op1_addr = OP1_REG_ADDR();
4547 op2_info = OP2_INFO();
4548 op2_addr = OP2_REG_ADDR();
4549 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4550 break;
4551 }
4552 if (opline->opcode == ZEND_ADD &&
4553 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4554 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4555 /* pass */
4556 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4557 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4558 break;
4559 }
4560 if (orig_op1_type != IS_UNKNOWN
4561 && (orig_op1_type & IS_TRACE_REFERENCE)
4562 && opline->op1_type == IS_CV
4563 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4564 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4565 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4566 goto jit_failure;
4567 }
4568 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4569 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4570 }
4571 } else {
4572 CHECK_OP1_TRACE_TYPE();
4573 }
4574 if (orig_op2_type != IS_UNKNOWN
4575 && (orig_op2_type & IS_TRACE_REFERENCE)
4576 && opline->op2_type == IS_CV
4577 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4578 if (!zend_jit_fetch_reference(&ctx, opline, orig_op2_type, &op2_info, &op2_addr,
4579 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4580 goto jit_failure;
4581 }
4582 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4583 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4584 }
4585 } else {
4586 CHECK_OP2_TRACE_TYPE();
4587 }
4588 res_addr = RES_REG_ADDR();
4589 if (Z_MODE(res_addr) != IS_REG
4590 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4591 send_result = 1;
4592 res_use_info = -1;
4593 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4594 if (!zend_jit_reuse_ip(&ctx)) {
4595 goto jit_failure;
4596 }
4597 } else {
4598 res_use_info = zend_jit_trace_type_to_info(
4599 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4600 if (opline->result_type == IS_CV) {
4601 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4602 }
4603 }
4604 res_info = RES_INFO();
4605 if (opline->opcode == ZEND_ADD &&
4606 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4607 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4608 if (!zend_jit_add_arrays(&ctx, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4609 goto jit_failure;
4610 }
4611 } else {
4612 bool may_overflow = (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa);
4613
4614 if (ra
4615 && may_overflow
4616 && ((res_info & MAY_BE_GUARD)
4617 && (res_info & MAY_BE_ANY) == MAY_BE_LONG)
4618 && ((opline->opcode == ZEND_ADD
4619 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)
4620 || (opline->opcode == ZEND_SUB
4621 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) {
4622 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
4623 }
4624 if (!zend_jit_math(&ctx, opline,
4625 op1_info, op1_addr,
4626 op2_info, op2_addr,
4627 res_use_info, res_info, res_addr,
4628 may_overflow,
4629 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4630 goto jit_failure;
4631 }
4632 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4633 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4634 && has_concrete_type(op1_info)
4635 && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
4636 && has_concrete_type(op2_info)
4637 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4638 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4639 }
4640 }
4641 goto done;
4642 case ZEND_CONCAT:
4643 case ZEND_FAST_CONCAT:
4644 op1_info = OP1_INFO();
4645 CHECK_OP1_TRACE_TYPE();
4646 op2_info = OP2_INFO();
4647 CHECK_OP2_TRACE_TYPE();
4648 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4649 break;
4650 }
4651 if (!(op1_info & MAY_BE_STRING) ||
4652 !(op2_info & MAY_BE_STRING)) {
4653 break;
4654 }
4655 res_addr = RES_REG_ADDR();
4656 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4657 send_result = 1;
4658 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4659 if (!zend_jit_reuse_ip(&ctx)) {
4660 goto jit_failure;
4661 }
4662 }
4663 if (!zend_jit_concat(&ctx, opline,
4664 op1_info, op2_info, res_addr,
4665 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4666 goto jit_failure;
4667 }
4668 goto done;
4669 case ZEND_ASSIGN_OP:
4670 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4671 break;
4672 }
4673 op1_info = OP1_INFO();
4674 CHECK_OP1_TRACE_TYPE();
4675 op2_info = OP2_INFO();
4676 CHECK_OP2_TRACE_TYPE();
4677 if (!zend_jit_supported_binary_op(
4678 opline->extended_value, op1_info, op2_info)) {
4679 break;
4680 }
4681 op1_addr = OP1_REG_ADDR();
4682 if (Z_MODE(op1_addr) != IS_REG
4683 || Z_LOAD(op1_addr)
4684 || Z_STORE(op1_addr)) {
4685 op1_mem_info = op1_info;
4686 } else {
4687 op1_mem_info = zend_jit_trace_type_to_info(
4688 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)));
4689 }
4690 op1_def_info = OP1_DEF_INFO();
4691 if (op1_def_info & MAY_BE_GUARD
4692 && !has_concrete_type(op1_def_info)) {
4693 op1_def_info &= ~MAY_BE_GUARD;
4694 }
4695 if (!zend_jit_assign_op(&ctx, opline,
4696 op1_info, op1_addr, OP1_RANGE(),
4697 op1_def_info, OP1_DEF_REG_ADDR(), op1_mem_info,
4698 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
4699 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4700 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4701 goto jit_failure;
4702 }
4703 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4704 && has_concrete_type(op1_info)
4705 && has_concrete_type(op2_info)) {
4706 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4707 if (opline->result_type != IS_UNUSED) {
4708 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4709 }
4710 }
4711 goto done;
4712 case ZEND_ASSIGN_DIM_OP:
4713 if (opline->result_type != IS_UNUSED) {
4714 break;
4715 }
4716 if (!zend_jit_supported_binary_op(
4717 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4718 break;
4719 }
4720 if (opline->op1_type == IS_CV
4721 && (opline+1)->op1_type == IS_CV
4722 && (opline+1)->op1.var == opline->op1.var) {
4723 /* skip $a[x] += $a; */
4724 break;
4725 }
4726 op1_info = OP1_INFO();
4727 op1_addr = OP1_REG_ADDR();
4728 op1_indirect = 0;
4729 if (opline->op1_type == IS_VAR) {
4730 if (orig_op1_type != IS_UNKNOWN
4731 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4732 op1_indirect = 1;
4733 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4734 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4735 goto jit_failure;
4736 }
4737 }
4738 }
4739 if (orig_op1_type != IS_UNKNOWN
4740 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4741 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4742 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4743 goto jit_failure;
4744 }
4745 if (opline->op1_type == IS_CV
4746 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4747 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4748 }
4749 } else {
4750 CHECK_OP1_TRACE_TYPE();
4751 }
4752 op2_info = OP2_INFO();
4753 CHECK_OP2_TRACE_TYPE();
4754 op1_data_info = OP1_DATA_INFO();
4755 CHECK_OP1_DATA_TRACE_TYPE();
4756 op1_def_info = OP1_DEF_INFO();
4757 if (!zend_jit_assign_dim_op(&ctx, opline,
4758 op1_info, op1_def_info, op1_addr, op1_indirect,
4759 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
4760 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
4761 op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(), val_type,
4762 zend_jit_trace_may_throw(opline, ssa_op, op_array, ssa,
4763 op1_info, op2_info, op1_data_info, val_type))) {
4764 goto jit_failure;
4765 }
4766 if (opline->op1_type == IS_VAR && !(op1_info & (MAY_BE_ANY-MAY_BE_NULL))) {
4767 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_ARRAY, 1);
4768 }
4769 goto done;
4770 case ZEND_PRE_INC_OBJ:
4771 case ZEND_PRE_DEC_OBJ:
4772 case ZEND_POST_INC_OBJ:
4773 case ZEND_POST_DEC_OBJ:
4774 if (opline->op2_type != IS_CONST
4775 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4776 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4777 break;
4778 }
4779 ce = NULL;
4780 ce_is_instanceof = 0;
4781 on_this = delayed_fetch_this = 0;
4782 op1_indirect = 0;
4783 if (opline->op1_type == IS_UNUSED) {
4784 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4785 ce = op_array->scope;
4786 /* scope is NULL for closures. */
4787 if (ce) {
4788 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4789 }
4790 op1_addr = 0;
4791 on_this = 1;
4792 } else {
4793 if (ssa_op->op1_use >= 0) {
4794 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4795 }
4796 op1_info = OP1_INFO();
4797 if (!(op1_info & MAY_BE_OBJECT)) {
4798 break;
4799 }
4800 op1_addr = OP1_REG_ADDR();
4801 if (opline->op1_type == IS_VAR) {
4802 if (orig_op1_type != IS_UNKNOWN
4803 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4804 op1_indirect = 1;
4805 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4806 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4807 goto jit_failure;
4808 }
4809 }
4810 }
4811 if (orig_op1_type != IS_UNKNOWN
4812 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4813 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4814 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4815 goto jit_failure;
4816 }
4817 if (opline->op1_type == IS_CV
4818 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4819 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4820 }
4821 } else {
4822 CHECK_OP1_TRACE_TYPE();
4823 }
4824 if (!(op1_info & MAY_BE_OBJECT)) {
4825 break;
4826 }
4827 if (ssa->var_info && ssa->ops) {
4828 if (ssa_op->op1_use >= 0) {
4829 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4830 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4831 ce = op1_ssa->ce;
4832 ce_is_instanceof = op1_ssa->is_instanceof;
4833 }
4834 }
4835 }
4836 if (delayed_fetch_this) {
4837 on_this = 1;
4838 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4839 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4840 } else if (op_array_ssa->ops
4841 && op_array_ssa->vars
4842 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4843 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4844 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4845 }
4846 }
4847 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
4848 op1_info, op1_addr,
4849 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4850 val_type)) {
4851 goto jit_failure;
4852 }
4853 goto done;
4854 case ZEND_ASSIGN_OBJ_OP:
4855 if (opline->result_type != IS_UNUSED) {
4856 break;
4857 }
4858 if (opline->op2_type != IS_CONST
4859 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4860 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4861 break;
4862 }
4863 if (opline->op1_type == IS_CV
4864 && (opline+1)->op1_type == IS_CV
4865 && (opline+1)->op1.var == opline->op1.var) {
4866 /* skip $a->prop += $a; */
4867 break;
4868 }
4869 if (!zend_jit_supported_binary_op(
4870 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4871 break;
4872 }
4873 ce = NULL;
4874 ce_is_instanceof = 0;
4875 on_this = delayed_fetch_this = 0;
4876 op1_indirect = 0;
4877 if (opline->op1_type == IS_UNUSED) {
4878 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4879 ce = op_array->scope;
4880 /* scope is NULL for closures. */
4881 if (ce) {
4882 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4883 }
4884 op1_addr = 0;
4885 on_this = 1;
4886 } else {
4887 if (ssa_op->op1_use >= 0) {
4888 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4889 }
4890 op1_info = OP1_INFO();
4891 if (!(op1_info & MAY_BE_OBJECT)) {
4892 break;
4893 }
4894 op1_addr = OP1_REG_ADDR();
4895 if (opline->op1_type == IS_VAR) {
4896 if (orig_op1_type != IS_UNKNOWN
4897 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4898 op1_indirect = 1;
4899 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4900 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4901 goto jit_failure;
4902 }
4903 }
4904 }
4905 if (orig_op1_type != IS_UNKNOWN
4906 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4907 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4908 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4909 goto jit_failure;
4910 }
4911 if (opline->op1_type == IS_CV
4912 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4913 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4914 }
4915 } else {
4916 CHECK_OP1_TRACE_TYPE();
4917 }
4918 if (!(op1_info & MAY_BE_OBJECT)) {
4919 break;
4920 }
4921 if (ssa->var_info && ssa->ops) {
4922 if (ssa_op->op1_use >= 0) {
4923 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4924 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4925 ce = op1_ssa->ce;
4926 ce_is_instanceof = op1_ssa->is_instanceof;
4927 }
4928 }
4929 }
4930 if (delayed_fetch_this) {
4931 on_this = 1;
4932 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4933 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4934 } else if (op_array_ssa->ops
4935 && op_array_ssa->vars
4936 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4937 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4938 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4939 }
4940 }
4941 op1_data_info = OP1_DATA_INFO();
4942 CHECK_OP1_DATA_TRACE_TYPE();
4943 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
4944 op1_info, op1_addr, op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(),
4945 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4946 val_type)) {
4947 goto jit_failure;
4948 }
4949 goto done;
4950 case ZEND_ASSIGN_OBJ:
4951 if (opline->op2_type != IS_CONST
4952 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4953 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4954 break;
4955 }
4956 ce = NULL;
4957 ce_is_instanceof = 0;
4958 on_this = delayed_fetch_this = 0;
4959 op1_indirect = 0;
4960 if (opline->op1_type == IS_UNUSED) {
4961 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4962 ce = op_array->scope;
4963 /* scope is NULL for closures. */
4964 if (ce) {
4965 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4966 }
4967 op1_addr = 0;
4968 on_this = 1;
4969 } else {
4970 if (ssa_op->op1_use >= 0) {
4971 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4972 }
4973 op1_info = OP1_INFO();
4974 if (!(op1_info & MAY_BE_OBJECT)) {
4975 break;
4976 }
4977 op1_addr = OP1_REG_ADDR();
4978 if (opline->op1_type == IS_VAR) {
4979 if (orig_op1_type != IS_UNKNOWN
4980 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4981 op1_indirect = 1;
4982 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4983 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4984 goto jit_failure;
4985 }
4986 }
4987 }
4988 if (orig_op1_type != IS_UNKNOWN
4989 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4990 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4991 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4992 goto jit_failure;
4993 }
4994 if (opline->op1_type == IS_CV
4995 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4996 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4997 }
4998 } else {
4999 CHECK_OP1_TRACE_TYPE();
5000 }
5001 if (!(op1_info & MAY_BE_OBJECT)) {
5002 break;
5003 }
5004 if (ssa->var_info && ssa->ops) {
5005 if (ssa_op->op1_use >= 0) {
5006 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5007 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5008 ce = op1_ssa->ce;
5009 ce_is_instanceof = op1_ssa->is_instanceof;
5010 }
5011 }
5012 }
5013 if (delayed_fetch_this) {
5014 on_this = 1;
5015 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
5016 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
5017 } else if (op_array_ssa->ops
5018 && op_array_ssa->vars
5019 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
5020 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
5021 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
5022 }
5023 }
5024 op1_data_info = OP1_DATA_INFO();
5025 CHECK_OP1_DATA_TRACE_TYPE();
5026 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
5027 op1_info, op1_addr, op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_DEF_REG_ADDR(),
5028 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
5029 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
5030 val_type,
5031 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5032 goto jit_failure;
5033 }
5034 if ((opline+1)->op1_type == IS_CV
5035 && (ssa_op+1)->op1_def >= 0
5036 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
5037 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
5038 }
5039 goto done;
5040 case ZEND_ASSIGN_DIM:
5041 op1_info = OP1_INFO();
5042 op1_addr = OP1_REG_ADDR();
5043 op1_indirect = 0;
5044 if (opline->op1_type == IS_CV
5045 && (opline+1)->op1_type == IS_CV
5046 && (opline+1)->op1.var == opline->op1.var) {
5047 /* skip $a[x] = $a; */
5048 break;
5049 }
5050 if (opline->op1_type == IS_VAR) {
5051 if (orig_op1_type != IS_UNKNOWN
5052 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5053 op1_indirect = 1;
5054 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
5055 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5056 goto jit_failure;
5057 }
5058 }
5059 }
5060 if (orig_op1_type != IS_UNKNOWN
5061 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5062 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5063 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5064 goto jit_failure;
5065 }
5066 if (opline->op1_type == IS_CV
5067 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5068 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5069 }
5070 } else {
5071 CHECK_OP1_TRACE_TYPE();
5072 }
5073 op2_info = OP2_INFO();
5074 CHECK_OP2_TRACE_TYPE();
5075 op1_data_info = OP1_DATA_INFO();
5076 CHECK_OP1_DATA_TRACE_TYPE();
5077 if (!zend_jit_assign_dim(&ctx, opline,
5078 op1_info, op1_addr, op1_indirect,
5079 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
5080 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
5081 op1_data_info, OP1_DATA_REG_ADDR(),
5082 (ctx.ra && (ssa_op+1)->op1_def >= 0) ? OP1_DATA_DEF_REG_ADDR() : 0,
5083 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
5084 val_type,
5085 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5086 goto jit_failure;
5087 }
5088 if ((opline+1)->op1_type == IS_CV
5089 && (ssa_op+1)->op1_def >= 0
5090 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
5091 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
5092 }
5093 if (opline->op1_type == IS_VAR && !(op1_info & (MAY_BE_ANY-MAY_BE_NULL))) {
5094 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_ARRAY, 1);
5095 }
5096 goto done;
5097 case ZEND_ASSIGN:
5098 if (opline->op1_type != IS_CV) {
5099 break;
5100 }
5101 op2_addr = OP2_REG_ADDR();
5102 op2_info = OP2_INFO();
5103 zend_jit_addr ref_addr = 0;
5104
5105 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
5106 op2_def_addr = op2_addr;
5107 } else {
5108 op2_def_addr = OP2_DEF_REG_ADDR();
5109 }
5110 CHECK_OP2_TRACE_TYPE();
5111 op1_info = OP1_INFO();
5112 op1_def_info = OP1_DEF_INFO();
5113 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
5114 if (op1_type < IS_STRING
5115 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
5116 if (!zend_jit_scalar_type_guard(&ctx, opline, opline->op1.var)) {
5117 goto jit_failure;
5118 }
5119 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
5120 } else {
5121 CHECK_OP1_TRACE_TYPE();
5122 }
5123 }
5124 op1_addr = OP1_REG_ADDR();
5125 op1_def_addr = OP1_DEF_REG_ADDR();
5126 if (Z_MODE(op1_def_addr) != IS_REG &&
5127 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
5128 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
5129 /* type may be not set */
5130 op1_info |= MAY_BE_NULL;
5131 }
5132 if (orig_op1_type != IS_UNKNOWN) {
5133 if (orig_op1_type & IS_TRACE_REFERENCE) {
5134 if (!zend_jit_guard_reference(&ctx, opline, &op1_addr, &ref_addr,
5135 !ssa->var_info[ssa_op->op1_use].guarded_reference)) {
5136 goto jit_failure;
5137 }
5138 op1_info &= ~MAY_BE_REF;
5139 if (opline->op1_type == IS_CV
5140 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5141 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5142 }
5143 if (opline->result_type == IS_UNUSED) {
5144 res_addr = 0;
5145 } else {
5146 res_addr = RES_REG_ADDR();
5147 if (Z_MODE(res_addr) != IS_REG
5148 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5149 send_result = 1;
5150 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5151 if (!zend_jit_reuse_ip(&ctx)) {
5152 goto jit_failure;
5153 }
5154 }
5155 }
5156 op1_def_addr = op1_addr;
5157 op1_def_info &= ~MAY_BE_REF;
5158 } else if (op1_info & MAY_BE_REF) {
5159 if (!zend_jit_noref_guard(&ctx, opline, op1_addr)) {
5160 goto jit_failure;
5161 }
5162 op1_info &= ~MAY_BE_REF;
5163 op1_def_info &= ~MAY_BE_REF;
5164 }
5165 }
5166 if (opline->result_type == IS_UNUSED) {
5167 res_addr = 0;
5168 res_info = -1;
5169 } else {
5170 res_addr = RES_REG_ADDR();
5171 res_info = RES_INFO();
5172 if (Z_MODE(res_addr) != IS_REG
5173 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5174 send_result = 1;
5175 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5176 if (!zend_jit_reuse_ip(&ctx)) {
5177 goto jit_failure;
5178 }
5179 }
5180 }
5181 if (!zend_jit_assign(&ctx, opline,
5182 op1_info, op1_addr,
5183 op1_def_info, op1_def_addr,
5184 op2_info, op2_addr, op2_def_addr,
5185 res_info, res_addr,
5186 ref_addr,
5187 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5188 goto jit_failure;
5189 }
5190 if (ssa_op->op2_def >= 0
5191 && Z_MODE(op2_addr) == IS_REG
5192 && ssa->vars[ssa_op->op2_def].no_val) {
5193 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5194 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5195
5196 if (STACK_MEM_TYPE(stack, var_num) != type
5197 && ssa->vars[ssa_op->op2_def].use_chain < 0
5198 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5199 if (!zend_jit_store_type(&ctx, var_num, type)) {
5200 return 0;
5201 }
5202 SET_STACK_TYPE(stack, var_num, type, 1);
5203 }
5204 }
5205 if (opline->op2_type == IS_CV
5206 && ssa_op->op2_def >= 0
5207 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5208 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5209 }
5210 goto done;
5211 case ZEND_CAST:
5212 if (opline->extended_value != op1_type) {
5213 break;
5214 }
5215 ZEND_FALLTHROUGH;
5216 case ZEND_QM_ASSIGN:
5217 op1_addr = OP1_REG_ADDR();
5218 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5219 op1_def_addr = op1_addr;
5220 } else {
5221 op1_def_addr = OP1_DEF_REG_ADDR();
5222 }
5223 op1_info = OP1_INFO();
5224 CHECK_OP1_TRACE_TYPE();
5225 res_info = RES_INFO();
5226 res_use_info = zend_jit_trace_type_to_info(
5227 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
5228 if (opline->result_type == IS_CV) {
5229 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
5230 }
5231 res_addr = RES_REG_ADDR();
5232 if (Z_MODE(res_addr) != IS_REG &&
5233 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5234 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5235 /* type may be not set */
5236 res_use_info |= MAY_BE_NULL;
5237 }
5238 if (!zend_jit_qm_assign(&ctx, opline,
5239 op1_info, op1_addr, op1_def_addr,
5240 res_use_info, res_info, res_addr)) {
5241 goto jit_failure;
5242 }
5243 if (ssa_op->op1_def >= 0
5244 && Z_MODE(op1_addr) == IS_REG
5245 && ssa->vars[ssa_op->op1_def].no_val) {
5246 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5247 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5248
5249 if (STACK_MEM_TYPE(stack, var_num) != type
5250 && ssa->vars[ssa_op->op1_def].use_chain < 0
5251 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5252 if (!zend_jit_store_type(&ctx, var_num, type)) {
5253 return 0;
5254 }
5255 SET_STACK_TYPE(stack, var_num, type, 1);
5256 }
5257 }
5258 if (opline->op1_type == IS_CV
5259 && ssa_op->op1_def >= 0
5260 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5261 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5262 }
5263 goto done;
5264 case ZEND_INIT_FCALL:
5265 case ZEND_INIT_FCALL_BY_NAME:
5266 case ZEND_INIT_NS_FCALL_BY_NAME:
5267 frame_flags = TRACE_FRAME_MASK_NESTED;
5268 if (!zend_jit_init_fcall(&ctx, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5269 goto jit_failure;
5270 }
5271 goto done;
5272 case ZEND_SEND_VAL:
5273 case ZEND_SEND_VAL_EX:
5274 if (opline->op2_type == IS_CONST) {
5275 /* Named parameters not supported in JIT */
5276 break;
5277 }
5278 if (opline->opcode == ZEND_SEND_VAL_EX
5279 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5280 break;
5281 }
5282 op1_info = OP1_INFO();
5283 CHECK_OP1_TRACE_TYPE();
5284 if (!zend_jit_send_val(&ctx, opline,
5285 op1_info, OP1_REG_ADDR())) {
5286 goto jit_failure;
5287 }
5288 if (frame->call && frame->call->func) {
5289 if (opline->op1_type == IS_CONST) {
5290 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5291 } else if (op1_type != IS_UNKNOWN) {
5292 if (op1_type == IS_UNDEF) {
5293 op1_type = IS_NULL;
5294 }
5295 zend_jit_trace_send_type(opline, frame->call, op1_type);
5296 }
5297 }
5298 goto done;
5299 case ZEND_SEND_REF:
5300 if (opline->op2_type == IS_CONST) {
5301 /* Named parameters not supported in JIT */
5302 break;
5303 }
5304 op1_info = OP1_INFO();
5305 if (!zend_jit_send_ref(&ctx, opline, op_array,
5306 op1_info, 0)) {
5307 goto jit_failure;
5308 }
5309 if (opline->op1_type == IS_CV
5310 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5311 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5312 }
5313 goto done;
5314 case ZEND_SEND_VAR:
5315 case ZEND_SEND_VAR_EX:
5316 case ZEND_SEND_VAR_NO_REF:
5317 case ZEND_SEND_VAR_NO_REF_EX:
5318 case ZEND_SEND_FUNC_ARG:
5319 if (opline->op2_type == IS_CONST) {
5320 /* Named parameters not supported in JIT */
5321 break;
5322 }
5323 if ((opline->opcode == ZEND_SEND_VAR_EX
5324 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5325 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5326 break;
5327 }
5328 op1_addr = OP1_REG_ADDR();
5329 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5330 op1_def_addr = op1_addr;
5331 } else {
5332 op1_def_addr = OP1_DEF_REG_ADDR();
5333 }
5334 op1_info = OP1_INFO();
5335 CHECK_OP1_TRACE_TYPE();
5336 if (!zend_jit_send_var(&ctx, opline, op_array,
5337 op1_info, op1_addr, op1_def_addr)) {
5338 goto jit_failure;
5339 }
5340 if (ssa_op->op1_def >= 0
5341 && Z_MODE(op1_addr) == IS_REG
5342 && ssa->vars[ssa_op->op1_def].no_val) {
5343 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5344 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5345
5346 if (STACK_MEM_TYPE(stack, var_num) != type
5347 && ssa->vars[ssa_op->op1_def].use_chain < 0
5348 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5349 if (!zend_jit_store_type(&ctx, var_num, type)) {
5350 return 0;
5351 }
5352 SET_STACK_TYPE(stack, var_num, type, 1);
5353 }
5354 }
5355 if (opline->op1_type == IS_CV
5356 && ssa_op->op1_def >= 0
5357 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5358 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5359 }
5360 if (frame->call && frame->call->func) {
5361 if ((opline->opcode == ZEND_SEND_VAR_EX
5362 || opline->opcode == ZEND_SEND_FUNC_ARG)
5363 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5364 goto done;
5365 }
5366 if (op1_type != IS_UNKNOWN) {
5367 if (op1_type == IS_UNDEF) {
5368 op1_type = IS_NULL;
5369 }
5370 zend_jit_trace_send_type(opline, frame->call, op1_type);
5371 }
5372 }
5373 goto done;
5374 case ZEND_CHECK_FUNC_ARG:
5375 if (!JIT_G(current_frame)
5376 || !JIT_G(current_frame)->call
5377 || !JIT_G(current_frame)->call->func) {
5378 break;
5379 }
5380 if (opline->op2_type == IS_CONST
5381 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5382 /* Named parameters not supported in JIT */
5383 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5384 break;
5385 }
5386 if (!zend_jit_check_func_arg(&ctx, opline)) {
5387 goto jit_failure;
5388 }
5389 goto done;
5390 case ZEND_CHECK_UNDEF_ARGS:
5391 if (JIT_G(current_frame)
5392 && JIT_G(current_frame)->call) {
5393 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5394 }
5395 if (!zend_jit_check_undef_args(&ctx, opline)) {
5396 goto jit_failure;
5397 }
5398 goto done;
5399 case ZEND_DO_UCALL:
5400 case ZEND_DO_ICALL:
5401 case ZEND_DO_FCALL_BY_NAME:
5402 case ZEND_DO_FCALL:
5403 if (!zend_jit_do_fcall(&ctx, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5404 goto jit_failure;
5405 }
5406 goto done;
5407 case ZEND_IS_EQUAL:
5408 case ZEND_IS_NOT_EQUAL:
5409 case ZEND_IS_SMALLER:
5410 case ZEND_IS_SMALLER_OR_EQUAL:
5411 case ZEND_CASE:
5412 op1_info = OP1_INFO();
5413 op2_info = OP2_INFO();
5414 skip_comparison =
5415 ssa_op != ssa->ops &&
5416 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5417 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5418 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5419 CHECK_OP1_TRACE_TYPE();
5420 CHECK_OP2_TRACE_TYPE();
5421 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5422 bool exit_if_true = 0;
5423 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5424 uint32_t exit_point;
5425
5426 if (ra) {
5427 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5428 }
5429 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5430 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5431 if (!exit_addr) {
5432 goto jit_failure;
5433 }
5434 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5435 if (!zend_jit_cmp(&ctx, opline,
5436 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5437 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5438 RES_REG_ADDR(),
5439 zend_may_throw(opline, ssa_op, op_array, ssa),
5440 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5441 goto jit_failure;
5442 }
5443 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5444 } else {
5445 smart_branch_opcode = 0;
5446 exit_addr = NULL;
5447 if (!zend_jit_cmp(&ctx, opline,
5448 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5449 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5450 RES_REG_ADDR(),
5451 zend_may_throw(opline, ssa_op, op_array, ssa),
5452 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5453 goto jit_failure;
5454 }
5455 }
5456 goto done;
5457 case ZEND_IS_IDENTICAL:
5458 case ZEND_IS_NOT_IDENTICAL:
5459 case ZEND_CASE_STRICT:
5460 op1_info = OP1_INFO();
5461 op2_info = OP2_INFO();
5462 skip_comparison =
5463 ssa_op != ssa->ops &&
5464 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5465 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5466 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5467 CHECK_OP1_TRACE_TYPE();
5468 CHECK_OP2_TRACE_TYPE();
5469 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5470 bool exit_if_true = 0;
5471 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5472 uint32_t exit_point;
5473
5474 if (ra) {
5475 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5476 }
5477 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5478 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5479 if (!exit_addr) {
5480 goto jit_failure;
5481 }
5482 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5483 exit_if_true = !exit_if_true;
5484 }
5485 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5486 if (!zend_jit_identical(&ctx, opline,
5487 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5488 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5489 RES_REG_ADDR(),
5490 zend_may_throw(opline, ssa_op, op_array, ssa),
5491 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5492 goto jit_failure;
5493 }
5494 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5495 } else {
5496 smart_branch_opcode = 0;
5497 exit_addr = NULL;
5498 if (!zend_jit_identical(&ctx, opline,
5499 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5500 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5501 RES_REG_ADDR(),
5502 zend_may_throw(opline, ssa_op, op_array, ssa),
5503 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5504 goto jit_failure;
5505 }
5506 }
5507 goto done;
5508 case ZEND_DEFINED:
5509 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5510 bool exit_if_true = 0;
5511 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5512 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5513
5514 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5515 if (!exit_addr) {
5516 goto jit_failure;
5517 }
5518 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5519 } else {
5520 smart_branch_opcode = 0;
5521 exit_addr = NULL;
5522 }
5523 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5524 goto jit_failure;
5525 }
5526 goto done;
5527 case ZEND_TYPE_CHECK:
5528 if (opline->extended_value == MAY_BE_RESOURCE) {
5529 // TODO: support for is_resource() ???
5530 break;
5531 }
5532 op1_info = OP1_INFO();
5533 CHECK_OP1_TRACE_TYPE();
5534 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5535 bool exit_if_true = 0;
5536 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5537 uint32_t exit_point;
5538
5539 if (ra) {
5540 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5541 }
5542 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5543 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5544 if (!exit_addr) {
5545 goto jit_failure;
5546 }
5547 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5548 } else {
5549 smart_branch_opcode = 0;
5550 exit_addr = NULL;
5551 }
5552 if (!zend_jit_type_check(&ctx, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5553 goto jit_failure;
5554 }
5555 goto done;
5556 case ZEND_RETURN:
5557 op1_info = OP1_INFO();
5558 CHECK_OP1_TRACE_TYPE();
5559 if (opline->op1_type == IS_CONST) {
5560 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5561 } else if (op1_type != IS_UNKNOWN) {
5562 res_type = op1_type;
5563 if (res_type == IS_UNDEF) {
5564 res_type = IS_NULL;
5565 }
5566 }
5567 if (op_array->type == ZEND_EVAL_CODE
5568 // TODO: support for top-level code
5569 || !op_array->function_name
5570 // TODO: support for IS_UNDEF ???
5571 || (op1_info & MAY_BE_UNDEF)) {
5572 if (!zend_jit_trace_handler(&ctx, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5573 goto jit_failure;
5574 }
5575 } else {
5576 int j;
5577 int may_throw = 0;
5578 bool left_frame = 0;
5579
5580 if (!zend_jit_return(&ctx, opline, op_array,
5581 op1_info, OP1_REG_ADDR())) {
5582 goto jit_failure;
5583 }
5584 if (op_array->last_var > 100) {
5585 /* To many CVs to unroll */
5586 if (!zend_jit_free_cvs(&ctx)) {
5587 goto jit_failure;
5588 }
5589 left_frame = 1;
5590 }
5591 if (!left_frame) {
5592 for (j = 0 ; j < op_array->last_var; j++) {
5593 uint32_t info;
5594 uint8_t type;
5595
5596 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5597 type = STACK_TYPE(stack, j);
5598 info = zend_jit_trace_type_to_info_ex(type, info);
5599 if (opline->op1_type == IS_CV
5600 && EX_VAR_TO_NUM(opline->op1.var) == j
5601 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5602 if (JIT_G(current_frame)
5603 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5604 continue;
5605 } else {
5606 info |= MAY_BE_NULL;
5607 }
5608 }
5609 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5610 if (!left_frame) {
5611 left_frame = 1;
5612 if (!zend_jit_leave_frame(&ctx)) {
5613 goto jit_failure;
5614 }
5615 }
5616 if (!zend_jit_free_cv(&ctx, info, j)) {
5617 goto jit_failure;
5618 }
5619 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5620 if (info & MAY_BE_RC1) {
5621 may_throw = 1;
5622 }
5623 }
5624 }
5625 }
5626 }
5627 if (!zend_jit_leave_func(&ctx, op_array, opline, op1_info, left_frame,
5628 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5629 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5630 goto jit_failure;
5631 }
5632 }
5633 goto done;
5634 case ZEND_BOOL:
5635 case ZEND_BOOL_NOT:
5636 op1_info = OP1_INFO();
5637 CHECK_OP1_TRACE_TYPE();
5638 if (!zend_jit_bool_jmpznz(&ctx, opline,
5639 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5640 -1, -1,
5641 zend_may_throw(opline, ssa_op, op_array, ssa),
5642 opline->opcode, NULL)) {
5643 goto jit_failure;
5644 }
5645 goto done;
5646 case ZEND_JMPZ:
5647 case ZEND_JMPNZ:
5648 case ZEND_JMPZ_EX:
5649 case ZEND_JMPNZ_EX:
5650 op1_info = OP1_INFO();
5651 CHECK_OP1_TRACE_TYPE();
5652 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5653 const zend_op *exit_opline = NULL;
5654 uint32_t exit_point;
5655
5656 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5657 /* taken branch */
5658 if (opline->opcode == ZEND_JMPNZ_EX) {
5659 smart_branch_opcode = ZEND_JMPZ_EX;
5660 } else if (opline->opcode == ZEND_JMPZ_EX) {
5661 smart_branch_opcode = ZEND_JMPNZ_EX;
5662 } else if (opline->opcode == ZEND_JMPNZ) {
5663 smart_branch_opcode = ZEND_JMPZ;
5664 } else {
5665 smart_branch_opcode = ZEND_JMPNZ;
5666 }
5667 exit_opline = opline + 1;
5668 } else if ((p+1)->opline == opline + 1) {
5669 /* not taken branch */
5670 smart_branch_opcode = opline->opcode;
5671 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5672 } else {
5673 ZEND_UNREACHABLE();
5674 }
5675 if (ra) {
5676 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5677 }
5678 if (!(op1_info & MAY_BE_GUARD)
5679 && has_concrete_type(op1_info)
5680 && concrete_type(op1_info) <= IS_TRUE) {
5681 /* unconditional branch */
5682 exit_addr = NULL;
5683 } else if (opline->result_type == IS_TMP_VAR) {
5684 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5685
5686 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5687 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5688 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5689 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5690 if (!exit_addr) {
5691 goto jit_failure;
5692 }
5693 } else {
5694 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5695 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5696 if (!exit_addr) {
5697 goto jit_failure;
5698 }
5699 }
5700 } else {
5701 ZEND_UNREACHABLE();
5702 }
5703 if (opline->result_type == IS_UNDEF) {
5704 res_addr = 0;
5705 } else {
5706 res_addr = RES_REG_ADDR();
5707 }
5708 if (!zend_jit_bool_jmpznz(&ctx, opline,
5709 op1_info, OP1_REG_ADDR(), res_addr,
5710 -1, -1,
5711 zend_may_throw(opline, ssa_op, op_array, ssa),
5712 smart_branch_opcode, exit_addr)) {
5713 goto jit_failure;
5714 }
5715 goto done;
5716 case ZEND_JMP_FRAMELESS:
5717 op1_info = OP1_INFO();
5718 ZEND_ASSERT((p+1)->op == ZEND_JIT_TRACE_VM);
5719 const zend_op *exit_opline = NULL;
5720 uint32_t exit_point;
5721 zend_jmp_fl_result guard;
5722
5723 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5724 /* taken branch */
5725 guard = ZEND_JMP_FL_HIT;
5726 exit_opline = opline + 1;
5727 } else if ((p+1)->opline == opline + 1) {
5728 /* not taken branch */
5729 guard = ZEND_JMP_FL_MISS;
5730 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5731 } else {
5732 ZEND_UNREACHABLE();
5733 }
5734 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5735 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5736 if (!exit_addr) {
5737 goto jit_failure;
5738 }
5739 if (!zend_jit_jmp_frameless(&ctx, opline, exit_addr, guard)) {
5740 goto jit_failure;
5741 }
5742 goto done;
5743 case ZEND_ISSET_ISEMPTY_CV:
5744 if ((opline->extended_value & ZEND_ISEMPTY)) {
5745 // TODO: support for empty() ???
5746 break;
5747 }
5748 op1_info = OP1_INFO();
5749 op1_addr = OP1_REG_ADDR();
5750 if (orig_op1_type != IS_UNKNOWN
5751 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5752 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5753 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5754 goto jit_failure;
5755 }
5756 if (opline->op1_type == IS_CV
5757 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5758 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5759 }
5760 } else {
5761 CHECK_OP1_TRACE_TYPE();
5762 }
5763 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5764 bool exit_if_true = 0;
5765 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5766 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5767
5768 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5769 if (!exit_addr) {
5770 goto jit_failure;
5771 }
5772 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5773 } else {
5774 smart_branch_opcode = 0;
5775 exit_addr = NULL;
5776 }
5777 if (!zend_jit_isset_isempty_cv(&ctx, opline,
5778 op1_info, op1_addr,
5779 smart_branch_opcode, -1, -1, exit_addr)) {
5780 goto jit_failure;
5781 }
5782 goto done;
5783 case ZEND_IN_ARRAY:
5784 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5785 break;
5786 }
5787 op1_info = OP1_INFO();
5788 op1_addr = OP1_REG_ADDR();
5789 CHECK_OP1_TRACE_TYPE();
5790 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5791 break;
5792 }
5793 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5794 bool exit_if_true = 0;
5795 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5796 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5797
5798 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5799 if (!exit_addr) {
5800 goto jit_failure;
5801 }
5802 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5803 } else {
5804 smart_branch_opcode = 0;
5805 exit_addr = NULL;
5806 }
5807 if (!zend_jit_in_array(&ctx, opline,
5808 op1_info, op1_addr,
5809 smart_branch_opcode, -1, -1, exit_addr)) {
5810 goto jit_failure;
5811 }
5812 goto done;
5813 case ZEND_FETCH_DIM_FUNC_ARG:
5814 if (!JIT_G(current_frame)
5815 || !JIT_G(current_frame)->call
5816 || !JIT_G(current_frame)->call->func
5817 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5818 break;
5819 }
5820 ZEND_FALLTHROUGH;
5821 case ZEND_FETCH_DIM_R:
5822 case ZEND_FETCH_DIM_IS:
5823 case ZEND_FETCH_LIST_R:
5824 op1_info = OP1_INFO();
5825 op1_addr = OP1_REG_ADDR();
5826 if (orig_op1_type != IS_UNKNOWN
5827 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5828 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5829 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5830 goto jit_failure;
5831 }
5832 if (opline->op1_type == IS_CV
5833 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5834 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5835 if (ssa_op->op1_def >= 0) {
5836 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5837 }
5838 }
5839 } else {
5840 CHECK_OP1_TRACE_TYPE();
5841 }
5842 op2_info = OP2_INFO();
5843 CHECK_OP2_TRACE_TYPE();
5844 res_info = RES_INFO();
5845 avoid_refcounting =
5846 ssa_op->op1_use >= 0 &&
5847 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5848 if (op1_info & MAY_BE_PACKED_GUARD) {
5849 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5850 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5851 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5852 && MAY_BE_PACKED(op1_info)
5853 && MAY_BE_HASH(op1_info)
5854 && orig_op1_type != IS_UNKNOWN) {
5855 op1_info |= MAY_BE_PACKED_GUARD;
5856 if (orig_op1_type & IS_TRACE_PACKED) {
5857 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5858 if (op1_type != IS_UNKNOWN) {
5859 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5860 }
5861 } else {
5862 op1_info &= ~MAY_BE_ARRAY_PACKED;
5863 if (op1_type != IS_UNKNOWN) {
5864 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5865 }
5866 }
5867 }
5868 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
5869 op1_info, op1_addr, avoid_refcounting,
5870 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
5871 res_info, RES_REG_ADDR(), val_type)) {
5872 goto jit_failure;
5873 }
5874 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5875 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5876 }
5877 goto done;
5878 case ZEND_FETCH_DIM_W:
5879 case ZEND_FETCH_DIM_RW:
5880 // case ZEND_FETCH_DIM_UNSET:
5881 case ZEND_FETCH_LIST_W:
5882 if (opline->op1_type != IS_CV
5883 && (orig_op1_type == IS_UNKNOWN
5884 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5885 break;
5886 }
5887 op1_info = OP1_INFO();
5888 op1_addr = OP1_REG_ADDR();
5889 if (opline->op1_type == IS_VAR) {
5890 if (orig_op1_type != IS_UNKNOWN
5891 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5892 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
5893 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5894 goto jit_failure;
5895 }
5896 } else {
5897 break;
5898 }
5899 }
5900 if (orig_op1_type != IS_UNKNOWN
5901 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5902 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5903 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5904 goto jit_failure;
5905 }
5906 if (opline->op1_type == IS_CV
5907 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5908 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5909 }
5910 } else {
5911 CHECK_OP1_TRACE_TYPE();
5912 }
5913 op2_info = OP2_INFO();
5914 CHECK_OP2_TRACE_TYPE();
5915 op1_def_info = OP1_DEF_INFO();
5916 if (!zend_jit_fetch_dim(&ctx, opline,
5917 op1_info, op1_addr,
5918 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
5919 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
5920 RES_REG_ADDR(), val_type)) {
5921 goto jit_failure;
5922 }
5923 if (ssa_op->result_def >= 0
5924 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5925 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5926 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5927 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5928 }
5929 goto done;
5930 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5931 if ((opline->extended_value & ZEND_ISEMPTY)) {
5932 // TODO: support for empty() ???
5933 break;
5934 }
5935 op1_info = OP1_INFO();
5936 op1_addr = OP1_REG_ADDR();
5937 if (orig_op1_type != IS_UNKNOWN
5938 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5939 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5940 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5941 goto jit_failure;
5942 }
5943 if (opline->op1_type == IS_CV
5944 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5945 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5946 }
5947 } else {
5948 CHECK_OP1_TRACE_TYPE();
5949 }
5950 op2_info = OP2_INFO();
5951 CHECK_OP2_TRACE_TYPE();
5952 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5953 bool exit_if_true = 0;
5954 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5955 uint32_t exit_point;
5956 int32_t old_ref = 0;
5957 uint8_t old_flags = 0;
5958
5959 if (ra) {
5960 if (opline->op2_type != IS_CONST) {
5961 old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->op2.var));
5962 old_flags = STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op2.var));
5963 }
5964 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5965 }
5966 if (ssa_op->op1_use >= 0
5967 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5968 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5969 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5970
5971 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5972 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5973 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5974 } else {
5975 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5976 }
5977 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5978 if (!exit_addr) {
5979 goto jit_failure;
5980 }
5981 if (old_ref) {
5982 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op2.var), old_ref, old_flags);
5983 }
5984 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5985 } else {
5986 smart_branch_opcode = 0;
5987 exit_addr = NULL;
5988 }
5989 avoid_refcounting =
5990 ssa_op->op1_use >= 0 &&
5991 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5992 if (op1_info & MAY_BE_PACKED_GUARD) {
5993 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5994 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5995 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5996 && MAY_BE_PACKED(op1_info)
5997 && MAY_BE_HASH(op1_info)
5998 && orig_op1_type != IS_UNKNOWN) {
5999 op1_info |= MAY_BE_PACKED_GUARD;
6000 if (orig_op1_type & IS_TRACE_PACKED) {
6001 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
6002 } else {
6003 op1_info &= ~MAY_BE_ARRAY_PACKED;
6004 }
6005 }
6006 if (!zend_jit_isset_isempty_dim(&ctx, opline,
6007 op1_info, op1_addr, avoid_refcounting,
6008 op2_info, OP2_REG_ADDR(), OP2_RANGE(), val_type,
6009 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
6010 smart_branch_opcode, -1, -1,
6011 exit_addr)) {
6012 goto jit_failure;
6013 }
6014 goto done;
6015 case ZEND_FETCH_OBJ_FUNC_ARG:
6016 if (!JIT_G(current_frame)
6017 || !JIT_G(current_frame)->call
6018 || !JIT_G(current_frame)->call->func
6019 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
6020 break;
6021 }
6022 ZEND_FALLTHROUGH;
6023 case ZEND_FETCH_OBJ_R:
6024 case ZEND_FETCH_OBJ_IS:
6025 case ZEND_FETCH_OBJ_W:
6026 on_this = delayed_fetch_this = 0;
6027 avoid_refcounting = 0;
6028 if (opline->op2_type != IS_CONST
6029 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
6030 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
6031 break;
6032 }
6033 ce = NULL;
6034 ce_is_instanceof = 0;
6035 op1_indirect = 0;
6036 if (opline->op1_type == IS_UNUSED) {
6037 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6038 ce = op_array->scope;
6039 /* scope is NULL for closures. */
6040 if (ce) {
6041 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
6042 }
6043 op1_addr = 0;
6044 on_this = 1;
6045 } else {
6046 op1_info = OP1_INFO();
6047 if (!(op1_info & MAY_BE_OBJECT)) {
6048 break;
6049 }
6050 op1_addr = OP1_REG_ADDR();
6051 if (opline->op1_type == IS_VAR
6052 && opline->opcode == ZEND_FETCH_OBJ_W) {
6053 if (orig_op1_type != IS_UNKNOWN
6054 && (orig_op1_type & IS_TRACE_INDIRECT)) {
6055 op1_indirect = 1;
6056 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
6057 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
6058 goto jit_failure;
6059 }
6060 }
6061 }
6062 if (orig_op1_type != IS_UNKNOWN
6063 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6064 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6065 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6066 goto jit_failure;
6067 }
6068 if (opline->op1_type == IS_CV
6069 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6070 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
6071 }
6072 } else {
6073 CHECK_OP1_TRACE_TYPE();
6074 }
6075 if (!(op1_info & MAY_BE_OBJECT)) {
6076 break;
6077 }
6078 if (ssa->var_info && ssa->ops) {
6079 if (ssa_op->op1_use >= 0) {
6080 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6081 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6082 ce = op1_ssa->ce;
6083 ce_is_instanceof = op1_ssa->is_instanceof;
6084 }
6085 }
6086 }
6087 if (ssa_op->op1_use >= 0) {
6088 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6089 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
6090 }
6091 if (delayed_fetch_this) {
6092 on_this = 1;
6093 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6094 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6095 } else if (op_array_ssa->ops
6096 && op_array_ssa->vars
6097 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6098 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6099 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6100 }
6101 }
6102 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
6103 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
6104 on_this, delayed_fetch_this, avoid_refcounting, op1_ce,
6105 RES_REG_ADDR(), val_type,
6106 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
6107 goto jit_failure;
6108 }
6109 goto done;
6110 case ZEND_FETCH_STATIC_PROP_FUNC_ARG:
6111 if (!JIT_G(current_frame)
6112 || !JIT_G(current_frame)->call
6113 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
6114 break;
6115 }
6116 ZEND_FALLTHROUGH;
6117 case ZEND_FETCH_STATIC_PROP_R:
6118 case ZEND_FETCH_STATIC_PROP_IS:
6119 case ZEND_FETCH_STATIC_PROP_W:
6120 case ZEND_FETCH_STATIC_PROP_RW:
6121 case ZEND_FETCH_STATIC_PROP_UNSET:
6122 if (!(opline->op1_type == IS_CONST
6123 && (opline->op2_type == IS_CONST
6124 || (opline->op2_type == IS_UNUSED
6125 && ((opline->op2.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF
6126 || (opline->op2.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT))))) {
6127 break;
6128 }
6129 if (!zend_jit_fetch_static_prop(&ctx, opline, op_array)) {
6130 goto jit_failure;
6131 }
6132 goto done;
6133 case ZEND_BIND_GLOBAL:
6134 orig_opline = opline;
6135 orig_ssa_op = ssa_op;
6136 while (1) {
6137 if (!ssa->ops || !ssa->var_info) {
6138 op1_info = MAY_BE_ANY|MAY_BE_REF;
6139 } else {
6140 op1_info = OP1_INFO();
6141 }
6142 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
6143 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
6144 }
6145 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
6146 goto jit_failure;
6147 }
6148 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
6149 opline++;
6150 ssa_op++;
6151 } else {
6152 break;
6153 }
6154 }
6155 opline = orig_opline;
6156 ssa_op = orig_ssa_op;
6157 goto done;
6158 case ZEND_RECV:
6159 if (!zend_jit_recv(&ctx, opline, op_array)) {
6160 goto jit_failure;
6161 }
6162 goto done;
6163 case ZEND_RECV_INIT:
6164 orig_opline = opline;
6165 orig_ssa_op = ssa_op;
6166 while (1) {
6167 if (!zend_jit_recv_init(&ctx, opline, op_array,
6168 (opline + 1)->opcode != ZEND_RECV_INIT,
6169 zend_may_throw(opline, ssa_op, op_array, ssa))) {
6170 goto jit_failure;
6171 }
6172 if ((opline+1)->opcode == ZEND_RECV_INIT) {
6173 opline++;
6174 ssa_op++;
6175 } else {
6176 break;
6177 }
6178 }
6179 opline = orig_opline;
6180 ssa_op = orig_ssa_op;
6181 goto done;
6182 case ZEND_FREE:
6183 case ZEND_FE_FREE:
6184 op1_info = OP1_INFO();
6185 if (!zend_jit_free(&ctx, opline, op1_info,
6186 zend_may_throw(opline, ssa_op, op_array, ssa))) {
6187 goto jit_failure;
6188 }
6189 goto done;
6190 case ZEND_ECHO:
6191 op1_info = OP1_INFO();
6192 CHECK_OP1_TRACE_TYPE();
6193 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6194 break;
6195 }
6196 if (!zend_jit_echo(&ctx, opline, op1_info)) {
6197 goto jit_failure;
6198 }
6199 goto done;
6200 case ZEND_STRLEN:
6201 op1_info = OP1_INFO();
6202 op1_addr = OP1_REG_ADDR();
6203 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
6204 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6205 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6206 goto jit_failure;
6207 }
6208 if (opline->op1_type == IS_CV
6209 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6210 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6211 }
6212 } else {
6213 CHECK_OP1_TRACE_TYPE();
6214 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6215 break;
6216 }
6217 }
6218 if (!zend_jit_strlen(&ctx, opline, op1_info, op1_addr, RES_REG_ADDR())) {
6219 goto jit_failure;
6220 }
6221 goto done;
6222 case ZEND_COUNT:
6223 op1_info = OP1_INFO();
6224 op1_addr = OP1_REG_ADDR();
6225 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
6226 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6227 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6228 goto jit_failure;
6229 }
6230 if (opline->op1_type == IS_CV
6231 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6232 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6233 }
6234 } else {
6235 CHECK_OP1_TRACE_TYPE();
6236 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6237 break;
6238 }
6239 }
6240 if (!zend_jit_count(&ctx, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6241 goto jit_failure;
6242 }
6243 goto done;
6244 case ZEND_FETCH_THIS:
6245 delayed_fetch_this = 0;
6246 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6247 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6248 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6249 delayed_fetch_this = 1;
6250 }
6251 }
6252 if (!zend_jit_fetch_this(&ctx, opline, op_array, delayed_fetch_this)) {
6253 goto jit_failure;
6254 }
6255 goto done;
6256 case ZEND_SWITCH_LONG:
6257 case ZEND_SWITCH_STRING:
6258 case ZEND_MATCH:
6259 if (!zend_jit_switch(&ctx, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6260 goto jit_failure;
6261 }
6262 goto done;
6263 case ZEND_VERIFY_RETURN_TYPE:
6264 if (opline->op1_type == IS_UNUSED) {
6265 /* Always throws */
6266 break;
6267 }
6268 if (opline->op1_type == IS_CONST) {
6269 /* TODO Different instruction format, has return value */
6270 break;
6271 }
6272 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6273 /* Not worth bothering with */
6274 break;
6275 }
6276 op1_info = OP1_INFO();
6277 CHECK_OP1_TRACE_TYPE();
6278 if (op1_info & MAY_BE_REF) {
6279 /* TODO May need reference unwrapping. */
6280 break;
6281 }
6282 if (!zend_jit_verify_return_type(&ctx, opline, op_array, op1_info)) {
6283 goto jit_failure;
6284 }
6285 goto done;
6286 case ZEND_FE_RESET_R:
6287 op1_info = OP1_INFO();
6288 CHECK_OP1_TRACE_TYPE();
6289 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6290 break;
6291 }
6292 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
6293 goto jit_failure;
6294 }
6295 goto done;
6296 case ZEND_FE_FETCH_R:
6297 op1_info = OP1_INFO();
6298 CHECK_OP1_TRACE_TYPE();
6299 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6300 break;
6301 }
6302 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6303 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6304 uint32_t exit_point;
6305
6306 if ((p+1)->opline == exit_opline) {
6307 /* taken branch (exit from loop) */
6308 exit_opline = opline;
6309 smart_branch_opcode = ZEND_NOP;
6310 } else if ((p+1)->opline == opline + 1) {
6311 /* not taken branch (loop) */
6312 smart_branch_opcode = ZEND_JMP;
6313 } else {
6314 ZEND_UNREACHABLE();
6315 }
6316 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6317 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6318 if (!exit_addr) {
6319 goto jit_failure;
6320 }
6321 } else {
6322 ZEND_UNREACHABLE();
6323 }
6324 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
6325 -1, smart_branch_opcode, exit_addr)) {
6326 goto jit_failure;
6327 }
6328 goto done;
6329 case ZEND_FETCH_CONSTANT:
6330 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6331 goto jit_failure;
6332 }
6333 goto done;
6334 case ZEND_INIT_METHOD_CALL:
6335 if (opline->op2_type != IS_CONST
6336 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6337 break;
6338 }
6339 on_this = delayed_fetch_this = 0;
6340 ce = NULL;
6341 ce_is_instanceof = 0;
6342 if (opline->op1_type == IS_UNUSED) {
6343 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6344 ce = op_array->scope;
6345 /* scope is NULL for closures. */
6346 if (ce) {
6347 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
6348 }
6349 op1_addr = 0;
6350 on_this = 1;
6351 } else {
6352 op1_info = OP1_INFO();
6353 op1_addr = OP1_REG_ADDR();
6354 if (polymorphic_side_trace) {
6355 op1_info = MAY_BE_OBJECT;
6356 } else if (orig_op1_type != IS_UNKNOWN
6357 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6358 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6359 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6360 goto jit_failure;
6361 }
6362 if (opline->op1_type == IS_CV
6363 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6364 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6365 }
6366 } else {
6367 CHECK_OP1_TRACE_TYPE();
6368 }
6369 if (ssa->var_info && ssa->ops) {
6370 if (ssa_op->op1_use >= 0) {
6371 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6372 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6373 ce = op1_ssa->ce;
6374 ce_is_instanceof = op1_ssa->is_instanceof;
6375 }
6376 }
6377 }
6378 if (ssa_op->op1_use >= 0) {
6379 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6380 }
6381 if (delayed_fetch_this) {
6382 on_this = 1;
6383 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6384 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6385 } else if (op_array_ssa->ops
6386 && op_array_ssa->vars
6387 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6388 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6389 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6390 }
6391 }
6392 frame_flags = TRACE_FRAME_MASK_NESTED;
6393 if (!zend_jit_init_method_call(&ctx, opline,
6394 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6395 op_array, ssa, ssa_op, frame->call_level,
6396 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6397 p + 1, peek_checked_stack - checked_stack,
6398 polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg : -1,
6399 polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg : -1,
6400 polymorphic_side_trace)) {
6401 goto jit_failure;
6402 }
6403 goto done;
6404 case ZEND_INIT_STATIC_METHOD_CALL:
6405 if (!(opline->op2_type == IS_CONST
6406 && (opline->op1_type == IS_CONST
6407 || (opline->op1_type == IS_UNUSED
6408 && ((opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF
6409 || (opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT))))) {
6410 break;
6411 }
6412 if (!zend_jit_init_static_method_call(&ctx, opline,
6413 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6414 op_array, ssa, ssa_op, frame->call_level,
6415 p + 1, peek_checked_stack - checked_stack)) {
6416 goto jit_failure;
6417 }
6418 goto done;
6419 case ZEND_INIT_DYNAMIC_CALL:
6420 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6421 break;
6422 }
6423 op2_info = OP2_INFO();
6424 CHECK_OP2_TRACE_TYPE();
6425 frame_flags = TRACE_FRAME_MASK_NESTED;
6426 if (!zend_jit_init_closure_call(&ctx, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6427 goto jit_failure;
6428 }
6429 goto done;
6430 case ZEND_SEND_ARRAY:
6431 case ZEND_SEND_UNPACK:
6432 if (JIT_G(current_frame)
6433 && JIT_G(current_frame)->call) {
6434 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6435 }
6436 break;
6437 case ZEND_ROPE_INIT:
6438 case ZEND_ROPE_ADD:
6439 case ZEND_ROPE_END:
6440 op2_info = OP2_INFO();
6441 CHECK_OP2_TRACE_TYPE();
6442 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6443 break;
6444 }
6445 if (!zend_jit_rope(&ctx, opline, op2_info)) {
6446 goto jit_failure;
6447 }
6448 goto done;
6449 case ZEND_FRAMELESS_ICALL_0:
6450 jit_frameless_icall0(jit, opline);
6451 goto done;
6452 case ZEND_FRAMELESS_ICALL_1:
6453 op1_info = OP1_INFO();
6454 jit_frameless_icall1(jit, opline, op1_info);
6455 goto done;
6456 case ZEND_FRAMELESS_ICALL_2:
6457 op1_info = OP1_INFO();
6458 op2_info = OP2_INFO();
6459 jit_frameless_icall2(jit, opline, op1_info, op2_info);
6460 goto done;
6461 case ZEND_FRAMELESS_ICALL_3:
6462 op1_info = OP1_INFO();
6463 op2_info = OP2_INFO();
6464 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
6465 goto done;
6466 default:
6467 break;
6468 }
6469 }
6470
6471 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6472 gen_handler = 1;
6473 op1_info = OP1_INFO();
6474 op2_info = OP2_INFO();
6475 if (op1_info & MAY_BE_GUARD) {
6476 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6477 }
6478 if (op2_info & MAY_BE_GUARD) {
6479 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6480 }
6481 if (!zend_jit_trace_handler(&ctx, op_array, opline,
6482 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6483 goto jit_failure;
6484 }
6485 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
6486 if (opline->opcode == ZEND_NEW && opline->result_type != IS_UNUSED) {
6487 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6488 }
6489 if (zend_jit_may_be_polymorphic_call(opline) ||
6490 zend_jit_may_be_modified((p+1)->func, op_array)) {
6491 if (!zend_jit_init_fcall_guard(&ctx, 0, (p+1)->func, opline+1)) {
6492 goto jit_failure;
6493 }
6494 }
6495 }
6496 }
6497
6498 done:
6499 polymorphic_side_trace = 0;
6500 switch (opline->opcode) {
6501 case ZEND_DO_FCALL:
6502 case ZEND_DO_ICALL:
6503 case ZEND_DO_UCALL:
6504 case ZEND_DO_FCALL_BY_NAME:
6505 case ZEND_CALLABLE_CONVERT:
6506 frame->call_level--;
6507 }
6508
6509 if (ra) {
6510 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
6511 }
6512
6513 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6514 && STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS)) {
6515 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6516 }
6517
6518 if (opline->opcode == ZEND_ROPE_INIT) {
6519 /* clear stack slots used by rope */
6520 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6521 uint32_t count =
6522 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6523
6524 do {
6525 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6526 var++;
6527 count--;
6528 } while (count);
6529 }
6530
6531 if (ssa_op) {
6532 zend_ssa_range tmp;
6533
6534 /* Keep information about known types on abstract stack */
6535 if (ssa_op->result_def >= 0) {
6536 uint8_t type = IS_UNKNOWN;
6537
6538 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6539 || send_result) {
6540 /* we didn't set result variable */
6541 type = IS_UNKNOWN;
6542 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6543 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6544 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6545 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6546 if (opline->op1_type != IS_CONST) {
6547 /* copy */
6548 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6549 }
6550 } else if (opline->opcode == ZEND_ASSIGN) {
6551 if (opline->op2_type != IS_CONST
6552 && ssa_op->op1_use >= 0
6553 /* assignment to typed reference may cause conversion */
6554 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6555 /* copy */
6556 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6557 }
6558 } else if (opline->opcode == ZEND_POST_INC
6559 || opline->opcode == ZEND_POST_DEC) {
6560 /* copy */
6561 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6562 }
6563 if (opline->opcode == ZEND_JMP_SET
6564 || opline->opcode == ZEND_COALESCE
6565 || opline->opcode == ZEND_JMP_NULL) {
6566 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6567 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6568 } else if ((p+1)->opline != (opline + 1)) {
6569 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6570 }
6571 } else {
6572 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6573 (gen_handler || type == IS_UNKNOWN || !ra || !RA_HAS_REG(ssa_op->result_def)));
6574
6575 if (op_array->last_live_range
6576 && opline->result.var > op_array->last_var
6577 && STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) != type) {
6578 if (!gen_handler && type != IS_UNKNOWN && ra && RA_HAS_REG(ssa_op->result_def)) {
6579 uint32_t var_num = opline->result.var;
6580 uint32_t op_num = opline - op_array->opcodes;
6581 const zend_live_range *range = op_array->live_range;
6582 int j;
6583
6584 op_num += zend_jit_trace_op_len(opline);
6585 for (j = 0; j < op_array->last_live_range; range++, j++) {
6586 if (range->start > op_num) {
6587 /* further blocks will not be relevant... */
6588 break;
6589 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
6590 /* check if opcodes in range may throw */
6591 bool store_type = 0;
6592 const zend_ssa_op *next_ssa_op = ssa_op + zend_jit_trace_op_len(opline);
6593 const zend_jit_trace_rec *q = p + 1;
6594
6595 while (1) {
6596 if (q->op != ZEND_JIT_TRACE_VM) {
6597 store_type = 1;
6598 break;
6599 }
6600 op_num = q->opline - op_array->opcodes;
6601 if (op_num >= range->end || op_num < range->start) {
6602 break;
6603 }
6604 if (zend_may_throw(q->opline, next_ssa_op, op_array, ssa)) {
6605 store_type = 1;
6606 break;
6607 }
6608 next_ssa_op += zend_jit_trace_op_len(q->opline);
6609 q++;
6610 }
6611 if (store_type) {
6612 var_num = EX_VAR_TO_NUM(var_num);
6613
6614 if (!zend_jit_store_type(&ctx, var_num, type)) {
6615 return 0;
6616 }
6617 SET_STACK_TYPE(stack, var_num, type, 1);
6618 }
6619 break;
6620 }
6621 }
6622 }
6623 }
6624 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6625 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6626 }
6627 if (type != IS_UNKNOWN) {
6628 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6629 if (opline->opcode == ZEND_FETCH_THIS
6630 && delayed_fetch_this) {
6631 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_THIS);
6632 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6633 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_ZVAL_ADDREF);
6634 } else if (ra && RA_HAS_REG(ssa_op->result_def)) {
6635 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def].ref,
6636 RA_REG_FLAGS(ssa_op->result_def) & ZREG_STORE);
6637 }
6638 }
6639 }
6640
6641 if (type == IS_LONG
6642 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) {
6643 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6644 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6645 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6646 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6647 ssa->var_info[ssa_op->result_def].has_range = 1;
6648 }
6649 }
6650 if (ssa_op->op1_def >= 0
6651 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6652 || opline->result_type != IS_CV
6653 || opline->result.var != opline->op1.var)) {
6654 uint8_t type = IS_UNKNOWN;
6655
6656 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6657 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6658 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6659 } else if (opline->opcode == ZEND_ASSIGN) {
6660 if (!(OP1_INFO() & MAY_BE_REF)
6661 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6662 if (opline->op2_type != IS_CONST) {
6663 /* copy */
6664 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6665 }
6666 }
6667 } else if (opline->opcode == ZEND_SEND_VAR
6668 || opline->opcode == ZEND_CAST
6669 || opline->opcode == ZEND_QM_ASSIGN
6670 || opline->opcode == ZEND_JMP_SET
6671 || opline->opcode == ZEND_COALESCE
6672 || opline->opcode == ZEND_JMP_NULL
6673 || opline->opcode == ZEND_FE_RESET_R) {
6674 /* keep old value */
6675 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6676 }
6677 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6678 (gen_handler || type == IS_UNKNOWN || !ra ||
6679 (!RA_HAS_REG(ssa_op->op1_def) &&
6680 !(ssa->vars[ssa_op->op1_def].no_val &&
6681 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6682 (opline->opcode == ZEND_QM_ASSIGN ||
6683 opline->opcode == ZEND_SEND_VAR ||
6684 opline->opcode == ZEND_SEND_VAR_EX ||
6685 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6686 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6687 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6688 if (type != IS_UNKNOWN) {
6689 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6690 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6691 uint8_t flags = RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE;
6692
6693 if (ssa_op->op1_use >= 0) {
6694 if (opline->opcode == ZEND_SEND_VAR
6695 || opline->opcode == ZEND_CAST
6696 || opline->opcode == ZEND_QM_ASSIGN
6697 || opline->opcode == ZEND_JMP_SET
6698 || opline->opcode == ZEND_COALESCE
6699 || opline->opcode == ZEND_JMP_NULL
6700 || opline->opcode == ZEND_FE_RESET_R) {
6701 if (!RA_HAS_REG(ssa_op->op1_use)) {
6702 flags |= ZREG_LOAD;
6703 }
6704 }
6705 }
6706 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref, flags);
6707 }
6708 }
6709 if (type == IS_LONG
6710 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6711 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6712 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6713 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6714 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6715 ssa->var_info[ssa_op->op1_def].has_range = 1;
6716 }
6717 }
6718 if (ssa_op->op2_def >= 0
6719 && (opline->opcode != ZEND_ASSIGN
6720 || opline->op1_type != IS_CV
6721 || opline->op1.var != opline->op2.var)) {
6722 uint8_t type = IS_UNKNOWN;
6723
6724 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6725 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6726 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6727 } else if (opline->opcode == ZEND_ASSIGN) {
6728 /* keep old value */
6729 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6730 }
6731 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6732 (gen_handler || type == IS_UNKNOWN || !ra ||
6733 (!RA_HAS_REG(ssa_op->op2_def) &&
6734 !(ssa->vars[ssa_op->op2_def].no_val &&
6735 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6736 opline->opcode == ZEND_ASSIGN))));
6737 if (type != IS_UNKNOWN) {
6738 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6739 if (ra && RA_HAS_REG(ssa_op->op2_def)) {
6740 uint8_t flags = RA_REG_FLAGS(ssa_op->op2_def) & ZREG_STORE;
6741
6742 if (ssa_op->op2_use >= 0) {
6743 if (opline->opcode == ZEND_ASSIGN) {
6744 if (!RA_HAS_REG(ssa_op->op2_use)
6745 ) {
6746 flags |= ZREG_LOAD;
6747 }
6748 }
6749 }
6750 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def].ref, flags);
6751 }
6752 }
6753 if (type == IS_LONG
6754 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) {
6755 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6756 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6757 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6758 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6759 ssa->var_info[ssa_op->op2_def].has_range = 1;
6760 }
6761 }
6762
6763 switch (opline->opcode) {
6764 case ZEND_ASSIGN_DIM:
6765 case ZEND_ASSIGN_OBJ:
6766 case ZEND_ASSIGN_STATIC_PROP:
6767 case ZEND_ASSIGN_DIM_OP:
6768 case ZEND_ASSIGN_OBJ_OP:
6769 case ZEND_ASSIGN_STATIC_PROP_OP:
6770 case ZEND_ASSIGN_OBJ_REF:
6771 case ZEND_ASSIGN_STATIC_PROP_REF:
6772 /* OP_DATA */
6773 ssa_op++;
6774 opline++;
6775 if (ssa_op->op1_def >= 0) {
6776 uint8_t type = IS_UNKNOWN;
6777
6778 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6779 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6780 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6781 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6782 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6783 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6784 /* keep old value */
6785 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6786 }
6787 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6788 (gen_handler || type == IS_UNKNOWN || !ra || !RA_HAS_REG(ssa_op->op1_def)));
6789 if (type != IS_UNKNOWN) {
6790 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6791 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6792 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref,
6793 RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE);
6794 }
6795 }
6796 if (type == IS_LONG
6797 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6798 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6799 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6800 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6801 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6802 ssa->var_info[ssa_op->op1_def].has_range = 1;
6803 }
6804 }
6805 ssa_op++;
6806 break;
6807 case ZEND_RECV_INIT:
6808 ssa_op++;
6809 opline++;
6810 while (opline->opcode == ZEND_RECV_INIT) {
6811 if (ssa_op->result_def >= 0) {
6812 uint8_t type = IS_UNKNOWN;
6813
6814 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6815 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6816 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6817 }
6818 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6819 (gen_handler || !ra || !RA_HAS_REG(ssa_op->result_def)));
6820 if (ra && RA_HAS_REG(ssa_op->result_def)) {
6821 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def].ref,
6822 RA_REG_FLAGS(ssa_op->result_def) & ZREG_STORE);
6823 }
6824 }
6825 ssa_op++;
6826 opline++;
6827 }
6828 break;
6829 case ZEND_BIND_GLOBAL:
6830 ssa_op++;
6831 opline++;
6832 while (opline->opcode == ZEND_BIND_GLOBAL) {
6833 if (ssa_op->op1_def >= 0) {
6834 uint8_t type = IS_UNKNOWN;
6835
6836 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6837 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6838 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6839 }
6840 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6841 (gen_handler || !ra || !RA_HAS_REG(ssa_op->op1_def)));
6842 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6843 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref,
6844 RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE);
6845 }
6846 }
6847 ssa_op++;
6848 opline++;
6849 }
6850 break;
6851 default:
6852 ssa_op += zend_jit_trace_op_len(opline);
6853 break;
6854 }
6855
6856 if (send_result) {
6857 ssa_op++;
6858 p++;
6859 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6860 p++;
6861 }
6862 send_result = 0;
6863 }
6864 }
6865 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6866 call = frame->call;
6867 assert(call && &call->func->op_array == p->op_array);
6868
6869 if (opline->opcode == ZEND_DO_UCALL
6870 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6871 || opline->opcode == ZEND_DO_FCALL) {
6872
6873 frame->call_opline = opline;
6874
6875 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6876 if (opline > op_array->opcodes) {
6877 const zend_op *prev_opline = opline - 1;
6878
6879 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6880 prev_opline--;
6881 }
6882 JIT_G(current_frame) = call;
6883 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6884 || prev_opline->opcode == ZEND_SEND_UNPACK
6885 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6886 && p->op_array->num_args
6887 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6888 && ((p+1)->op == ZEND_JIT_TRACE_VM
6889 || (p+1)->op == ZEND_JIT_TRACE_END)
6890 && (TRACE_FRAME_NUM_ARGS(call) < 0
6891 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6892 && !zend_jit_trace_opline_guard(&ctx, (p+1)->opline)) {
6893 goto jit_failure;
6894 }
6895 JIT_G(current_frame) = frame;
6896 }
6897 }
6898
6899 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6900 p++;
6901 break;
6902 }
6903 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6904 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6905 TRACE_FRAME_SET_THIS_CHECKED(call);
6906 }
6907 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6908 TRACE_FRAME_SET_THIS_CHECKED(call);
6909 }
6910 op_array = (zend_op_array*)p->op_array;
6911 ctx.current_op_array = op_array;
6912 jit_extension =
6913 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6914 op_array_ssa = &jit_extension->func_info.ssa;
6915 frame->call = call->prev;
6916 call->prev = frame;
6917 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6918 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6919 } else {
6920 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6921 }
6922 JIT_G(current_frame) = frame = call;
6923 stack = frame->stack;
6924 if (ra) {
6925 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6926
6927 for (i = 0; i < op_array->last_var; i++, j++) {
6928 if (RA_HAS_REG(j) && (RA_REG_FLAGS(j) & ZREG_LOAD) != 0) {
6929 if ((ssa->var_info[j].type & MAY_BE_GUARD) != 0) {
6930 uint8_t op_type;
6931
6932 ssa->var_info[j].type &= ~MAY_BE_GUARD;
6933 op_type = concrete_type(ssa->var_info[j].type);
6934 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), op_type)) {
6935 goto jit_failure;
6936 }
6937 SET_STACK_TYPE(stack, i, op_type, 1);
6938 }
6939 if (!zend_jit_load_var(&ctx, ssa->var_info[j].type, i, j)) {
6940 goto jit_failure;
6941 }
6942 SET_STACK_REF_EX(stack, i, ra[j].ref, ZREG_LOAD);
6943 }
6944 }
6945 }
6946 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6947 op_array = (zend_op_array*)p->op_array;
6948 ctx.current_op_array = op_array;
6949 jit_extension =
6950 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6951 op_array_ssa = &jit_extension->func_info.ssa;
6952 top = frame;
6953 if (frame->prev) {
6954 checked_stack = frame->old_checked_stack;
6955 peek_checked_stack = frame->old_peek_checked_stack;
6956 frame = frame->prev;
6957 stack = frame->stack;
6958 ZEND_ASSERT(&frame->func->op_array == op_array);
6959 } else {
6960 frame = zend_jit_trace_ret_frame(frame, op_array);
6961 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6962 frame->used_stack = checked_stack = peek_checked_stack = 0;
6963 stack = frame->stack;
6964 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6965 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6966
6967 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6968 /* Initialize abstract stack using SSA */
6969 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6970 && has_concrete_type(ssa->var_info[j].type)) {
6971 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6972 } else {
6973 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6974 }
6975 }
6976 if (ra) {
6977 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6978 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6979 if (RA_HAS_REG(j) && (RA_REG_FLAGS(j) & ZREG_LOAD) != 0) {
6980 if (!zend_jit_load_var(&ctx, ssa->var_info[j].type, i, j)) {
6981 goto jit_failure;
6982 }
6983 SET_STACK_REF_EX(stack, i, ra[j].ref, ZREG_LOAD);
6984 }
6985 }
6986 }
6987 } else {
6988 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6989 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6990 }
6991 }
6992 opline = NULL;
6993 }
6994 JIT_G(current_frame) = frame;
6995 if (res_type != IS_UNKNOWN
6996 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6997 const zend_op *opline = (p+1)->opline - 1;
6998 if (opline->result_type != IS_UNUSED) {
6999 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
7000 }
7001 }
7002 res_type = IS_UNKNOWN;
7003 } else if (p->op == ZEND_JIT_TRACE_END) {
7004 break;
7005 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7006 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
7007 int num_args = -1;
7008
7009 if (init_opline
7010 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
7011 num_args = init_opline->extended_value;
7012 }
7013
7014 call = top;
7015 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
7016 call->prev = frame->call;
7017 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
7018 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
7019 if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
7020 TRACE_FRAME_SET_CLOSURE_CALL(call);
7021 }
7022 }
7023 if (init_opline) {
7024 if (init_opline->opcode != ZEND_NEW
7025 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
7026 || init_opline->op1_type == IS_UNDEF
7027 || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
7028 && ssa_op
7029 && (ssa_op-1)->op1_use >=0
7030 && ssa->var_info[(ssa_op-1)->op1_use].delayed_fetch_this))
7031 && (init_opline->opcode != ZEND_INIT_USER_CALL
7032 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
7033 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
7034 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
7035 ) {
7036 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
7037 } else if (init_opline->opcode == ZEND_NEW
7038 || (init_opline->opcode == ZEND_INIT_METHOD_CALL
7039 && init_opline->op1_type != IS_UNDEF
7040 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
7041 && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) {
7042 TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call);
7043 }
7044 }
7045 frame->call = call;
7046 top = zend_jit_trace_call_frame(top, p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info));
7047 if (p->func) {
7048 if (p->func->type == ZEND_USER_FUNCTION) {
7049 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
7050 zend_jit_op_array_trace_extension *jit_extension =
7051 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
7052
7053 i = 0;
7054 while (i < p->op_array->num_args) {
7055 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
7056 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7057 i++;
7058 }
7059 while (i < p->op_array->last_var) {
7060 if (jit_extension
7061 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
7062 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7063 } else {
7064 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
7065 }
7066 i++;
7067 }
7068 while (i < p->op_array->last_var + p->op_array->T) {
7069 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7070 i++;
7071 }
7072 } else {
7073 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
7074 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7075 }
7076 }
7077 } else {
7078 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
7079 for (i = 0; i < p->op_array->num_args; i++) {
7080 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7081 }
7082 }
7083 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
7084 int skip_guard = 0;
7085
7086 if (init_opline) {
7087 zend_call_info *call_info = jit_extension->func_info.callee_info;
7088
7089 while (call_info) {
7090 if (call_info->caller_init_opline == init_opline
7091 && !call_info->is_prototype) {
7092 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
7093 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
7094 && init_opline->op1_type != IS_CONST) {
7095 break;
7096 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
7097 break;
7098 }
7099 }
7100 skip_guard = 1;
7101 break;
7102 }
7103 call_info = call_info->next_callee;
7104 }
7105 if (!skip_guard
7106 && !zend_jit_may_be_polymorphic_call(init_opline)
7107 && !zend_jit_may_be_modified(p->func, op_array)) {
7108 skip_guard = 1;
7109 }
7110 }
7111
7112 if (!skip_guard) {
7113 if (!opline) {
7114 zend_jit_trace_rec *q = p + 1;
7115 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
7116 q++;
7117 }
7118 opline = q->opline;
7119 ZEND_ASSERT(opline != NULL);
7120 }
7121 if (!zend_jit_init_fcall_guard(&ctx,
7122 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
7123 goto jit_failure;
7124 }
7125 }
7126 }
7127 }
7128 call->old_checked_stack = checked_stack;
7129 call->old_peek_checked_stack = peek_checked_stack;
7130 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
7131 frame->call_level++;
7132 call->used_stack = checked_stack = peek_checked_stack = 0;
7133 } else {
7134 if (p->func) {
7135 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
7136 } else {
7137 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
7138 }
7139 switch (init_opline->opcode) {
7140 case ZEND_INIT_FCALL:
7141 case ZEND_INIT_FCALL_BY_NAME:
7142 case ZEND_INIT_NS_FCALL_BY_NAME:
7143 case ZEND_INIT_METHOD_CALL:
7144 case ZEND_INIT_DYNAMIC_CALL:
7145 //case ZEND_INIT_STATIC_METHOD_CALL:
7146 //case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
7147 //case ZEND_INIT_USER_CALL:
7148 //case ZEND_NEW:
7149 checked_stack += call->used_stack;
7150 if (checked_stack > peek_checked_stack) {
7151 peek_checked_stack = checked_stack;
7152 }
7153 break;
7154 default:
7155 checked_stack = peek_checked_stack = 0;
7156 }
7157 }
7158 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7159 call = frame->call;
7160 if (call) {
7161 checked_stack = call->old_checked_stack;
7162 peek_checked_stack = call->old_peek_checked_stack;
7163 top = call;
7164 frame->call = call->prev;
7165 }
7166 } else {
7167 ZEND_UNREACHABLE();
7168 }
7169 }
7170
7171 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
7172
7173 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7174
7175 if (!parent_trace && zend_jit_trace_uses_initial_ip(&ctx)) {
7176 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
7177 }
7178
7179 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
7180 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7181 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7182 if (ra) {
7183 zend_ssa_phi *phi = ssa->blocks[1].phis;
7184
7185 while (phi) {
7186 if (RA_HAS_REG(phi->sources[1])
7187 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
7188 && (RA_REG_FLAGS(phi->sources[1]) & (ZREG_LOAD|ZREG_STORE)) == 0) {
7189
7190 if (!RA_HAS_REG(phi->ssa_var)
7191 || (RA_REG_FLAGS(phi->ssa_var) & (ZREG_LOAD|ZREG_STORE)) == 0) {
7192 /* Store actual type to memory to avoid deoptimization mistakes */
7193 zend_jit_store_var_type(&ctx, phi->var, STACK_TYPE(stack, phi->var));
7194 }
7195 }
7196 phi = phi->next;
7197 }
7198 }
7199 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7200 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7201 && !zend_jit_set_ip(&ctx, p->opline)) {
7202 goto jit_failure;
7203 }
7204 }
7205 t->link = ZEND_JIT_TRACE_NUM;
7206 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7207 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
7208 }
7209 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
7210 const void *timeout_exit_addr = NULL;
7211
7212 t->flags |= ZEND_JIT_TRACE_LOOP;
7213
7214 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7215 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7216 || (ra
7217 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
7218 /* Deoptimize to the first instruction of the loop */
7219 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
7220
7221 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7222 if (!timeout_exit_addr) {
7223 goto jit_failure;
7224 }
7225 } else {
7226 timeout_exit_addr = zend_jit_stub_handlers[jit_stub_interrupt_handler];
7227 }
7228 }
7229
7230 zend_jit_trace_end_loop(&ctx, jit->trace_loop_ref, timeout_exit_addr); /* jump back to start of the trace loop */
7231 }
7232 } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) {
7233 if (ra
7234 && (p-1)->op != ZEND_JIT_TRACE_ENTER
7235 && (p-1)->op != ZEND_JIT_TRACE_BACK
7236 && opline->opcode != ZEND_DO_UCALL
7237 && opline->opcode != ZEND_DO_FCALL
7238 && opline->opcode != ZEND_DO_FCALL_BY_NAME
7239 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
7240 for (i = 0; i < op_array->last_var + op_array->T; i++) {
7241 int32_t ref = STACK_REF(stack, i);
7242 uint8_t type = STACK_TYPE(stack, i);
7243
7244 if (ref && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
7245 if (!zend_jit_store_ref(jit, 1 << type, i, ref, STACK_MEM_TYPE(stack, i) != type)) {
7246 goto jit_failure;
7247 }
7248 SET_STACK_TYPE(stack, i, type, 1);
7249 } else if (i < op_array->last_var
7250 && type != IS_UNKNOWN
7251 && type != STACK_MEM_TYPE(stack, i)
7252 && zend_jit_trace_must_store_type(op_array, op_array_ssa, opline - op_array->opcodes, i, type)) {
7253 if (!zend_jit_store_type(jit, i, type)) {
7254 return 0;
7255 }
7256 SET_STACK_TYPE(stack, i, type, 1);
7257 }
7258 CLEAR_STACK_REF(stack, i);
7259 }
7260 }
7261 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
7262 const void *timeout_exit_addr = NULL;
7263
7264 t->link = zend_jit_find_trace(p->opline->handler);
7265 if (t->link == 0) {
7266 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
7267 * by zend_jit_trace_exit() in another thread after this
7268 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
7269 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
7270 * the "_counter_handler" functions, and these are not registered
7271 * tracer functions */
7272 goto jit_failure;
7273 }
7274 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7275 && !zend_jit_set_ip(&ctx, p->opline)) {
7276 goto jit_failure;
7277 }
7278 if (!parent_trace && zend_jit_trace_uses_initial_ip(&ctx)) {
7279 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
7280 }
7281 if (parent_trace
7282 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
7283 && zend_jit_traces[parent_trace].root == t->link) {
7284 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
7285 uint32_t exit_point;
7286
7287 for (i = 0; i < op_array->last_var + op_array->T; i++) {
7288 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
7289 }
7290 exit_point = zend_jit_trace_get_exit_point(zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM);
7291 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7292 if (!timeout_exit_addr) {
7293 goto jit_failure;
7294 }
7295 } else {
7296 timeout_exit_addr = zend_jit_stub_handlers[jit_stub_interrupt_handler];
7297 }
7298 }
7299 zend_jit_trace_link_to_root(&ctx, &zend_jit_traces[t->link], timeout_exit_addr);
7300 } else {
7301 zend_jit_trace_return(&ctx, 0, NULL);
7302 }
7303 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
7304 zend_jit_trace_return(&ctx, 0, NULL);
7305 } else {
7306 // TODO: not implemented ???
7307 ZEND_ASSERT(0 && p->stop);
7308 }
7309
7310 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7311 goto jit_failure;
7312 }
7313
7314 handler = zend_jit_finish(&ctx);
7315
7316 if (handler) {
7317 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
7318 const zend_op_array *rec_op_array;
7319
7320 rec_op_array = op_array = trace_buffer->op_array;
7321 jit_extension =
7322 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7323 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7324 for (;;p++) {
7325 if (p->op == ZEND_JIT_TRACE_VM) {
7326 opline = p->opline;
7327 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7328 if (p->op_array == rec_op_array) {
7329 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7330 }
7331 op_array = p->op_array;
7332 jit_extension =
7333 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7334 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7335 op_array = p->op_array;
7336 jit_extension =
7337 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7338 } else if (p->op == ZEND_JIT_TRACE_END) {
7339 break;
7340 }
7341 }
7342 } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) {
7343 if (opline
7344 && (opline->opcode == ZEND_DO_UCALL
7345 || opline->opcode == ZEND_DO_FCALL
7346 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7347 || opline->opcode == ZEND_YIELD
7348 || opline->opcode == ZEND_YIELD_FROM
7349 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7350 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7351 }
7352 if (JIT_G(current_frame)
7353 && JIT_G(current_frame)->prev) {
7354 frame = JIT_G(current_frame)->prev;
7355 do {
7356 if (frame->call_opline) {
7357 op_array = &frame->func->op_array;
7358 jit_extension =
7359 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7360 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7361 }
7362 frame = frame->prev;
7363 } while (frame);
7364 }
7365 }
7366 }
7367
7368 jit_failure:
7369 zend_jit_free_ctx(&ctx);
7370
7371 if (name) {
7372 zend_string_release(name);
7373 }
7374
7375 jit_cleanup:
7376 /* Clean up used op_arrays */
7377 while (num_op_arrays > 0) {
7378 op_array = op_arrays[--num_op_arrays];
7379 jit_extension =
7380 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7381
7382 jit_extension->func_info.num = 0;
7383 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7384 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7385 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7386 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7387 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7388 }
7389
7390 zend_arena_release(&CG(arena), checkpoint);
7391
7392 JIT_G(current_frame) = NULL;
7393 JIT_G(current_trace) = NULL;
7394
7395 return handler;
7396 }
7397
zend_jit_trace_escape_name(uint32_t trace_num,uint32_t exit_num)7398 static zend_string *zend_jit_trace_escape_name(uint32_t trace_num, uint32_t exit_num)
7399 {
7400 smart_str buf = {0};
7401
7402 smart_str_appends(&buf," ESCAPE-");
7403 smart_str_append_long(&buf, (zend_long)trace_num);
7404 smart_str_appendc(&buf, '-');
7405 smart_str_append_long(&buf, (zend_long)exit_num);
7406 smart_str_0(&buf);
7407 return buf.s;
7408 }
7409
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7410 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7411 {
7412 const void *handler = NULL;
7413 zend_jit_ctx ctx;
7414 zend_string *name;
7415 void *checkpoint;
7416 const zend_op *opline;
7417 uint32_t stack_size;
7418 zend_jit_trace_stack *stack;
7419 bool original_handler = 0;
7420
7421 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7422 return zend_jit_stub_handlers[jit_stub_trace_escape];
7423 }
7424
7425 name = zend_jit_trace_escape_name(trace_num, exit_num);
7426
7427 if (!zend_jit_deoptimizer_start(&ctx, name, trace_num, exit_num)) {
7428 zend_string_release(name);
7429 return NULL;
7430 }
7431
7432 checkpoint = zend_arena_checkpoint(CG(arena));;
7433
7434 /* Deoptimization */
7435 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7436 stack = zend_jit_traces[trace_num].exit_info[exit_num].stack_size ?
7437 zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset :
7438 NULL;
7439
7440 if (!zend_jit_trace_deoptimization(&ctx,
7441 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7442 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7443 stack, stack_size, NULL, NULL,
7444 zend_jit_traces[trace_num].constants,
7445 zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg,
7446 0)) {
7447 goto jit_failure;
7448 }
7449
7450 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7451 if (opline) {
7452 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7453 zend_jit_op_array_trace_extension *jit_extension =
7454 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(zend_jit_traces[zend_jit_traces[trace_num].root].op_array);
7455
7456 if (ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->orig_handler != opline->handler) {
7457 /* prevent endless loop */
7458 original_handler = 1;
7459 }
7460 }
7461 zend_jit_set_ip_ex(&ctx, opline, original_handler);
7462 }
7463
7464 zend_jit_trace_return(&ctx, original_handler, opline);
7465
7466 handler = zend_jit_finish(&ctx);
7467
7468 jit_failure:
7469 zend_jit_free_ctx(&ctx);
7470 zend_string_release(name);
7471 zend_arena_release(&CG(arena), checkpoint);
7472 return handler;
7473 }
7474
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7475 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7476 {
7477 zend_jit_trace_stop ret;
7478 const void *handler;
7479 uint8_t orig_trigger;
7480 zend_jit_trace_info *t = NULL;
7481 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7482 bool do_bailout = 0;
7483
7484 zend_shared_alloc_lock();
7485
7486 /* Checks under lock */
7487 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7488 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7489 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7490 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7491 } else {
7492 zend_try {
7493 SHM_UNPROTECT();
7494 zend_jit_unprotect();
7495
7496 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7497
7498 t->id = ZEND_JIT_TRACE_NUM;
7499 t->root = ZEND_JIT_TRACE_NUM;
7500 t->parent = 0;
7501 t->link = 0;
7502 t->exit_count = 0;
7503 t->child_count = 0;
7504 t->stack_map_size = 0;
7505 t->flags = 0;
7506 t->polymorphism = 0;
7507 t->jmp_table_size = 0;
7508 t->op_array = trace_buffer[0].op_array;
7509 t->opline = trace_buffer[1].opline;
7510 t->exit_info = exit_info;
7511 t->stack_map = NULL;
7512 t->consts_count = 0;
7513 t->constants = NULL;
7514
7515 orig_trigger = JIT_G(trigger);
7516 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7517
7518 handler = zend_jit_trace(trace_buffer, 0, 0);
7519
7520 JIT_G(trigger) = orig_trigger;
7521
7522 if (handler) {
7523 zend_jit_trace_exit_info *shared_exit_info = NULL;
7524
7525 t->exit_info = NULL;
7526 if (t->exit_count) {
7527 /* reallocate exit_info into shared memory */
7528 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7529 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7530
7531 if (!shared_exit_info) {
7532 if (t->stack_map) {
7533 efree(t->stack_map);
7534 t->stack_map = NULL;
7535 }
7536 if (t->constants) {
7537 efree(t->constants);
7538 t->constants = NULL;
7539 }
7540 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7541 goto exit;
7542 }
7543 memcpy(shared_exit_info, exit_info,
7544 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7545 t->exit_info = shared_exit_info;
7546 }
7547
7548 if (t->stack_map_size) {
7549 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7550 if (!shared_stack_map) {
7551 efree(t->stack_map);
7552 t->stack_map = NULL;
7553 if (t->constants) {
7554 efree(t->constants);
7555 t->constants = NULL;
7556 }
7557 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7558 goto exit;
7559 }
7560 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7561 efree(t->stack_map);
7562 t->stack_map = shared_stack_map;
7563 }
7564
7565 if (t->consts_count) {
7566 zend_jit_exit_const *constants = (zend_jit_exit_const*)zend_shared_alloc(t->consts_count * sizeof(zend_jit_exit_const));
7567 if (!constants) {
7568 efree(t->constants);
7569 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7570 goto exit;
7571 }
7572 memcpy(constants, t->constants, t->consts_count * sizeof(zend_jit_exit_const));
7573 efree(t->constants);
7574 t->constants = constants;
7575 }
7576
7577 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7578 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7579
7580 ((zend_op*)opline)->handler = handler;
7581
7582 ZEND_JIT_TRACE_NUM++;
7583 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7584
7585 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7586 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7587 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7588 if (t->stack_map) {
7589 efree(t->stack_map);
7590 t->stack_map = NULL;
7591 }
7592 if (t->constants) {
7593 efree(t->constants);
7594 t->constants = NULL;
7595 }
7596 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7597 } else {
7598 if (t->stack_map) {
7599 efree(t->stack_map);
7600 t->stack_map = NULL;
7601 }
7602 if (t->constants) {
7603 efree(t->constants);
7604 t->constants = NULL;
7605 }
7606 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7607 }
7608
7609 exit:;
7610 } zend_catch {
7611 do_bailout = 1;
7612 } zend_end_try();
7613
7614 zend_jit_protect();
7615 SHM_PROTECT();
7616 }
7617
7618 zend_shared_alloc_unlock();
7619
7620 if (do_bailout) {
7621 zend_bailout();
7622 }
7623
7624 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7625 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7626 && t->exit_count > 0) {
7627 zend_jit_dump_exit_info(t);
7628 }
7629
7630 return ret;
7631 }
7632
7633 /* Set counting handler back to original VM handler. */
zend_jit_stop_hot_trace_counters(zend_op_array * op_array)7634 static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array)
7635 {
7636 zend_jit_op_array_trace_extension *jit_extension;
7637 uint32_t i;
7638
7639 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7640 for (i = 0; i < op_array->last; i++) {
7641 /* Opline with Jit-ed code handler is skipped. */
7642 if (jit_extension->trace_info[i].trace_flags &
7643 (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7644 continue;
7645 }
7646 if (jit_extension->trace_info[i].trace_flags &
7647 (ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_START_RETURN)) {
7648 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7649 }
7650 }
7651 }
7652
7653 /* Get the tracing op_array. */
zend_jit_stop_persistent_op_array(zend_op_array * op_array)7654 static void zend_jit_stop_persistent_op_array(zend_op_array *op_array)
7655 {
7656 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
7657 if (!func_info) {
7658 return;
7659 }
7660 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
7661 zend_jit_stop_hot_trace_counters(op_array);
7662 }
7663 }
7664
7665 /* Get all op_arrays with counter handler. */
zend_jit_stop_persistent_script(zend_persistent_script * script)7666 static void zend_jit_stop_persistent_script(zend_persistent_script *script)
7667 {
7668 zend_class_entry *ce;
7669 zend_op_array *op_array;
7670
7671 zend_jit_stop_persistent_op_array(&script->script.main_op_array);
7672
7673 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
7674 zend_jit_stop_persistent_op_array(op_array);
7675 } ZEND_HASH_FOREACH_END();
7676
7677 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
7678 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
7679 if (op_array->type == ZEND_USER_FUNCTION) {
7680 zend_jit_stop_persistent_op_array(op_array);
7681 }
7682 } ZEND_HASH_FOREACH_END();
7683 } ZEND_HASH_FOREACH_END();
7684 }
7685
7686 /* Get all scripts which are accelerated by JIT */
zend_jit_stop_counter_handlers(void)7687 static void zend_jit_stop_counter_handlers(void)
7688 {
7689 if (ZCSG(jit_counters_stopped)) {
7690 return;
7691 }
7692
7693 zend_shared_alloc_lock();
7694 /* mprotect has an extreme overhead, avoid calls to it for every function. */
7695 SHM_UNPROTECT();
7696 if (!ZCSG(jit_counters_stopped)) {
7697 ZCSG(jit_counters_stopped) = true;
7698 for (uint32_t i = 0; i < ZCSG(hash).max_num_entries; i++) {
7699 zend_accel_hash_entry *cache_entry;
7700 for (cache_entry = ZCSG(hash).hash_table[i]; cache_entry; cache_entry = cache_entry->next) {
7701 zend_persistent_script *script;
7702 if (cache_entry->indirect) continue;
7703 script = (zend_persistent_script *)cache_entry->data;
7704 zend_jit_stop_persistent_script(script);
7705 }
7706 }
7707 }
7708 SHM_PROTECT();
7709 zend_shared_alloc_unlock();
7710 }
7711
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7712 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7713 {
7714 zend_shared_alloc_lock();
7715
7716 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7717 SHM_UNPROTECT();
7718 zend_jit_unprotect();
7719
7720 ((zend_op*)opline)->handler =
7721 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7722
7723 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7724
7725 zend_jit_protect();
7726 SHM_PROTECT();
7727 }
7728
7729 zend_shared_alloc_unlock();
7730 }
7731
zend_jit_blacklist_function(zend_op_array * op_array)7732 ZEND_EXT_API void zend_jit_blacklist_function(zend_op_array *op_array) {
7733 zend_jit_op_array_trace_extension *jit_extension = (zend_jit_op_array_trace_extension *)ZEND_FUNC_INFO(op_array);
7734 if (!jit_extension || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) {
7735 return;
7736 }
7737
7738 zend_shared_alloc_lock();
7739 SHM_UNPROTECT();
7740 zend_jit_unprotect();
7741
7742 zend_jit_stop_persistent_op_array(op_array);
7743 jit_extension->func_info.flags &= ~ZEND_FUNC_JIT_ON_HOT_TRACE;
7744
7745 zend_jit_protect();
7746 SHM_PROTECT();
7747 zend_shared_alloc_unlock();
7748 }
7749
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7750 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7751 {
7752 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7753 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7754 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7755 uint32_t cache_slot = JIT_G(bad_root_slot);
7756 uint32_t i;
7757
7758 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7759 if (cache_opline[i] == opline) {
7760 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7761 cache_opline[i] = NULL;
7762 return 1;
7763 } else {
7764 #if 0
7765 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7766 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7767 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7768 }
7769 #endif
7770 cache_count[i]++;
7771 cache_stop[i] = stop;
7772 return 0;
7773 }
7774 }
7775 }
7776 i = cache_slot;
7777 cache_opline[i] = opline;
7778 cache_count[i] = 1;
7779 cache_stop[i] = stop;
7780 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7781 JIT_G(bad_root_slot) = cache_slot;
7782 return 0;
7783 }
7784
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7785 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7786 {
7787 zend_jit_trace_rec *p = trace_buffer;
7788 const zend_op_array *op_array;
7789 const zend_op *opline;
7790 uint32_t level = 1 + trace_buffer[0].level;
7791 int idx, len, i, v, vars_count, call_level;
7792
7793 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7794 op_array = p->op_array;
7795 p += ZEND_JIT_TRACE_START_REC_SIZE;
7796 idx = 0;
7797 call_level = 0;
7798
7799 if (tssa && tssa->var_info) {
7800 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7801 vars_count = op_array->last_var;
7802 } else {
7803 vars_count = op_array->last_var + op_array->T;
7804 }
7805 for (i = 0; i < vars_count; i++) {
7806 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7807 fprintf(stderr, " %*c;", level, ' ');
7808 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7809 fprintf(stderr, "\n");
7810 }
7811 }
7812 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7813 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7814 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7815 zend_ssa_phi *p = tssa->blocks[1].phis;
7816
7817 fprintf(stderr, "LOOP:\n");
7818
7819 while (p) {
7820 fprintf(stderr, " ;");
7821 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7822 fprintf(stderr, " = Phi(");
7823 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7824 fprintf(stderr, ", ");
7825 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7826 fprintf(stderr, ")\n");
7827 p = p->next;
7828 }
7829 }
7830 }
7831
7832 while (1) {
7833 if (p->op == ZEND_JIT_TRACE_VM) {
7834 uint8_t op1_type, op2_type, op3_type;
7835
7836 opline = p->opline;
7837 fprintf(stderr, "%04d%*c",
7838 (int)(opline - op_array->opcodes),
7839 level, ' ');
7840 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7841
7842 op1_type = p->op1_type;
7843 op2_type = p->op2_type;
7844 op3_type = p->op3_type;
7845 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7846 fprintf(stderr, " ;");
7847 if (op1_type != IS_UNKNOWN) {
7848 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7849 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7850 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7851 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7852 p++;
7853 fprintf(stderr, " op1(%sobject of class %s)", ref,
7854 ZSTR_VAL(p->ce->name));
7855 } else {
7856 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7857 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7858 }
7859 }
7860 if (op2_type != IS_UNKNOWN) {
7861 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7862 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7863 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7864 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7865 p++;
7866 fprintf(stderr, " op2(%sobject of class %s)", ref,
7867 ZSTR_VAL(p->ce->name));
7868 } else {
7869 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7870 fprintf(stderr, " op2(%s%s)", ref, type);
7871 }
7872 }
7873 if (op3_type != IS_UNKNOWN) {
7874 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7875 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7876 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7877 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7878 fprintf(stderr, " op3(%s%s)", ref, type);
7879 }
7880 }
7881 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7882 uint8_t val_type;
7883 const char *type;
7884
7885 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7886 fprintf(stderr, " ;");
7887 }
7888 p++;
7889 val_type = p->op1_type;
7890
7891 if (val_type == IS_UNDEF) {
7892 type = "undef";
7893 } else if (val_type == IS_REFERENCE) {
7894 type = "ref";
7895 } else {
7896 type = zend_get_type_by_const(val_type);
7897 }
7898 fprintf(stderr, " val(%s)", type);
7899 }
7900 fprintf(stderr, "\n");
7901 idx++;
7902
7903 len = zend_jit_trace_op_len(opline);
7904 while (len > 1) {
7905 opline++;
7906 fprintf(stderr, "%04d%*c;",
7907 (int)(opline - op_array->opcodes),
7908 level, ' ');
7909 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7910 idx++;
7911 len--;
7912 fprintf(stderr, "\n");
7913 }
7914 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7915 op_array = p->op_array;
7916 fprintf(stderr, " %*c>enter %s%s%s\n",
7917 level, ' ',
7918 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7919 op_array->scope ? "::" : "",
7920 op_array->function_name ?
7921 ZSTR_VAL(op_array->function_name) :
7922 ZSTR_VAL(op_array->filename));
7923 level++;
7924 if (tssa && tssa->var_info) {
7925 call_level++;
7926 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7927 vars_count = op_array->last_var;
7928 for (i = 0; i < vars_count; i++, v++) {
7929 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7930 fprintf(stderr, " %*c;", level, ' ');
7931 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7932 fprintf(stderr, "\n");
7933 }
7934 }
7935 }
7936 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7937 op_array = p->op_array;
7938 level--;
7939 fprintf(stderr, " %*c<back %s%s%s\n",
7940 level, ' ',
7941 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7942 op_array->scope ? "::" : "",
7943 op_array->function_name ?
7944 ZSTR_VAL(op_array->function_name) :
7945 ZSTR_VAL(op_array->filename));
7946 if (tssa && tssa->var_info) {
7947 if (call_level == 0) {
7948 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7949 vars_count = op_array->last_var + op_array->T;
7950 for (i = 0; i < vars_count; i++, v++) {
7951 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7952 fprintf(stderr, " %*c;", level, ' ');
7953 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7954 fprintf(stderr, "\n");
7955 }
7956 }
7957 } else {
7958 call_level--;
7959 }
7960 }
7961 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7962 if (p->func != (zend_function*)&zend_pass_function) {
7963 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7964 level, ' ',
7965 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7966 (p->func && p->func->common.scope) ? "::" : "",
7967 (p->func && p->func->common.function_name) ? ZSTR_VAL(p->func->common.function_name) : "???");
7968 } else {
7969 fprintf(stderr, " %*c>skip\n",
7970 level, ' ');
7971 }
7972 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7973 if (p->func != (zend_function*)&zend_pass_function) {
7974 fprintf(stderr, " %*c>call %s%s%s\n",
7975 level, ' ',
7976 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7977 (p->func && p->func->common.scope) ? "::" : "",
7978 (p->func && p->func->common.function_name) ? ZSTR_VAL(p->func->common.function_name) : "???");
7979 } else {
7980 fprintf(stderr, " %*c>skip\n",
7981 level, ' ');
7982 }
7983 } else if (p->op == ZEND_JIT_TRACE_END) {
7984 break;
7985 }
7986 p++;
7987 }
7988 }
7989
zend_jit_dump_exit_info(zend_jit_trace_info * t)7990 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7991 {
7992 int i, j;
7993
7994 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7995 for (i = 0; i < t->exit_count; i++) {
7996 const zend_op_array *op_array = t->exit_info[i].op_array;
7997 uint32_t stack_size = t->exit_info[i].stack_size;
7998 zend_jit_trace_stack *stack = t->exit_info[i].stack_size ? t->stack_map + t->exit_info[i].stack_offset : NULL;
7999
8000 fprintf(stderr, " exit_%d:", i);
8001 if (t->exit_info[i].opline) {
8002 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
8003 } else {
8004 fprintf(stderr, " ----/");
8005 }
8006 if (t->exit_info[i].stack_size) {
8007 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
8008 } else {
8009 fprintf(stderr, "----/0");
8010 }
8011 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
8012 fprintf(stderr, "/VM");
8013 }
8014 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8015 fprintf(stderr, "/CALL");
8016 }
8017 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
8018 fprintf(stderr, "/POLY");
8019 if (t->exit_info[i].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8020 fprintf(stderr, "(%s, %s)",
8021 t->exit_info[i].poly_func_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_func_reg) : "?",
8022 t->exit_info[i].poly_this_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_this_reg) : "?");
8023 }
8024 }
8025 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
8026 fprintf(stderr, "/FREE_OP1");
8027 }
8028 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
8029 fprintf(stderr, "/FREE_OP2");
8030 }
8031 for (j = 0; j < stack_size; j++) {
8032 uint8_t type = STACK_TYPE(stack, j);
8033 if (type != IS_UNKNOWN) {
8034 fprintf(stderr, " ");
8035 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
8036 fprintf(stderr, ":");
8037 if (type == IS_UNDEF) {
8038 fprintf(stderr, "undef");
8039 } else {
8040 fprintf(stderr, "%s", zend_get_type_by_const(type));
8041 }
8042 if (STACK_FLAGS(stack, j) == ZREG_CONST) {
8043 if (type == IS_LONG) {
8044 fprintf(stderr, "(" ZEND_LONG_FMT ")", (zend_long)t->constants[STACK_REF(stack, j)].i);
8045 } else if (type == IS_DOUBLE) {
8046 fprintf(stderr, "(%g)", t->constants[STACK_REF(stack, j)].d);
8047 } else {
8048 ZEND_UNREACHABLE();
8049 }
8050 } else if (STACK_FLAGS(stack, j) == ZREG_TYPE_ONLY) {
8051 fprintf(stderr, "(type_only)");
8052 } else if (STACK_FLAGS(stack, j) == ZREG_THIS) {
8053 fprintf(stderr, "(this)");
8054 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_ADDREF) {
8055 fprintf(stderr, "(zval_try_addref)");
8056 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
8057 fprintf(stderr, "zval_copy(%s)", zend_reg_name(STACK_REG(stack, j)));
8058 } else if (STACK_FLAGS(stack, j) & ZREG_SPILL_SLOT) {
8059 if (STACK_REG(stack, j) == ZREG_NONE) {
8060 fprintf(stderr, "(spill=0x%x", STACK_REF(stack, j));
8061 } else {
8062 fprintf(stderr, "(spill=0x%x(%s)", STACK_REF(stack, j), zend_reg_name(STACK_REG(stack, j)));
8063 }
8064 if (STACK_FLAGS(stack, j) != 0) {
8065 fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
8066 }
8067 fprintf(stderr, ")");
8068 } else if (STACK_REG(stack, j) != ZREG_NONE) {
8069 fprintf(stderr, "(%s", zend_reg_name(STACK_REG(stack, j)));
8070 if (STACK_FLAGS(stack, j) != 0) {
8071 fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
8072 }
8073 fprintf(stderr, ")");
8074 }
8075 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_ADDREF) {
8076 fprintf(stderr, ":unknown(zval_try_addref)");
8077 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
8078 fprintf(stderr, " ");
8079 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
8080 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name(STACK_REG(stack, j)));
8081 }
8082 }
8083 fprintf(stderr, "\n");
8084 }
8085 }
8086
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)8087 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
8088 {
8089 const zend_op *orig_opline;
8090 zend_jit_trace_stop stop;
8091 int ret = 0;
8092 zend_op_array *op_array;
8093 zend_jit_op_array_trace_extension *jit_extension;
8094 size_t offset;
8095 uint32_t trace_num;
8096 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8097
8098 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8099 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
8100 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8101
8102 repeat:
8103 trace_num = ZEND_JIT_TRACE_NUM;
8104 orig_opline = opline;
8105 op_array = &EX(func)->op_array;
8106 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8107 offset = jit_extension->offset;
8108
8109 EX(opline) = opline;
8110
8111 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
8112 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
8113 return 0;
8114 }
8115
8116 if (JIT_G(tracing)) {
8117 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
8118 return 0;
8119 }
8120
8121 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8122 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8123 trace_num,
8124 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8125 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8126 EX(func)->op_array.scope ? "::" : "",
8127 EX(func)->op_array.function_name ?
8128 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8129 ZSTR_VAL(EX(func)->op_array.filename),
8130 opline->lineno);
8131 }
8132
8133 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8134 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8135 zend_jit_stop_counter_handlers();
8136 goto abort;
8137 }
8138
8139 JIT_G(tracing) = 1;
8140 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
8141 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0, 0);
8142 JIT_G(tracing) = 0;
8143
8144 if (stop & ZEND_JIT_TRACE_HALT) {
8145 ret = -1;
8146 }
8147 stop &= ~ZEND_JIT_TRACE_HALT;
8148
8149 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
8150 orig_opline = trace_buffer[1].opline;
8151 op_array = (zend_op_array*)trace_buffer[0].op_array;
8152 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8153 offset = jit_extension->offset;
8154 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8155 const zend_op_array *op_array = trace_buffer[0].op_array;
8156 const zend_op *opline = trace_buffer[1].opline;
8157 zend_jit_op_array_trace_extension *jit_extension =
8158 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8159 size_t offset = jit_extension->offset;
8160
8161 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8162 trace_num,
8163 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8164 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8165 op_array->scope ? "::" : "",
8166 op_array->function_name ?
8167 ZSTR_VAL(op_array->function_name) : "$main",
8168 ZSTR_VAL(op_array->filename),
8169 opline->lineno);
8170 }
8171 }
8172
8173 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8174 zend_jit_dump_trace(trace_buffer, NULL);
8175 }
8176
8177 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8178 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8179 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8180 uint32_t idx = trace_buffer[1].last;
8181 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
8182 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8183 trace_num,
8184 link_to);
8185 } else {
8186 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8187 trace_num,
8188 zend_jit_trace_stop_description[stop]);
8189 }
8190 }
8191 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
8192 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8193 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8194 fprintf(stderr, "---- TRACE %d %s\n",
8195 trace_num,
8196 zend_jit_trace_stop_description[stop]);
8197 }
8198 } else {
8199 goto abort;
8200 }
8201 } else {
8202 abort:
8203 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8204 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8205 trace_num,
8206 zend_jit_trace_stop_description[stop]);
8207 }
8208 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8209 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
8210 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8211 fprintf(stderr, "---- TRACE %d blacklisted\n",
8212 trace_num);
8213 }
8214 zend_jit_blacklist_root_trace(orig_opline, offset);
8215 }
8216 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8217 execute_data = EG(current_execute_data);
8218 opline = EX(opline);
8219 goto repeat;
8220 }
8221 }
8222
8223 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8224 fprintf(stderr, "\n");
8225 }
8226
8227 return ret;
8228 }
8229
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)8230 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
8231 {
8232 const void *handler;
8233 bool do_bailout = 0;
8234
8235 zend_shared_alloc_lock();
8236
8237 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
8238 SHM_UNPROTECT();
8239 zend_jit_unprotect();
8240
8241 zend_try {
8242 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
8243
8244 if (handler) {
8245 zend_jit_link_side_trace(
8246 zend_jit_traces[trace_num].code_start,
8247 zend_jit_traces[trace_num].code_size,
8248 zend_jit_traces[trace_num].jmp_table_size,
8249 exit_num,
8250 handler);
8251 }
8252 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
8253 } zend_catch {
8254 do_bailout = 1;
8255 } zend_end_try();
8256
8257 zend_jit_protect();
8258 SHM_PROTECT();
8259 }
8260
8261 zend_shared_alloc_unlock();
8262
8263 if (do_bailout) {
8264 zend_bailout();
8265 }
8266 }
8267
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)8268 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
8269 {
8270 uint8_t *counter = JIT_G(exit_counters) +
8271 zend_jit_traces[trace_num].exit_counters + exit_num;
8272
8273 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
8274 return 1;
8275 }
8276 (*counter)++;
8277 return 0;
8278 }
8279
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)8280 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
8281 {
8282 uint8_t *counter = JIT_G(exit_counters) +
8283 zend_jit_traces[trace_num].exit_counters + exit_num;
8284
8285 if (*counter + 1 >= JIT_G(hot_side_exit)) {
8286 return 1;
8287 }
8288 (*counter)++;
8289 return 0;
8290 }
8291
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)8292 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
8293 {
8294 zend_jit_trace_stop ret;
8295 const void *handler;
8296 uint8_t orig_trigger;
8297 zend_jit_trace_info *t;
8298 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
8299 bool do_bailout = 0;
8300
8301 zend_shared_alloc_lock();
8302
8303 /* Checks under lock */
8304 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8305 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
8306 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8307 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8308 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8309 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8310 } else {
8311 SHM_UNPROTECT();
8312 zend_jit_unprotect();
8313
8314 zend_try {
8315 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
8316
8317 t->id = ZEND_JIT_TRACE_NUM;
8318 t->root = zend_jit_traces[parent_num].root;
8319 t->parent = parent_num;
8320 t->link = 0;
8321 t->exit_count = 0;
8322 t->child_count = 0;
8323 t->stack_map_size = 0;
8324 t->flags = 0;
8325 t->polymorphism = polymorphism;
8326 t->jmp_table_size = 0;
8327 t->opline = NULL;
8328 t->exit_info = exit_info;
8329 t->stack_map = NULL;
8330 t->consts_count = 0;
8331 t->constants = NULL;
8332
8333 orig_trigger = JIT_G(trigger);
8334 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
8335
8336 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
8337
8338 JIT_G(trigger) = orig_trigger;
8339
8340 if (handler) {
8341 zend_jit_trace_exit_info *shared_exit_info = NULL;
8342
8343 t->exit_info = NULL;
8344 if (t->exit_count) {
8345 /* reallocate exit_info into shared memory */
8346 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
8347 sizeof(zend_jit_trace_exit_info) * t->exit_count);
8348
8349 if (!shared_exit_info) {
8350 if (t->stack_map) {
8351 efree(t->stack_map);
8352 t->stack_map = NULL;
8353 }
8354 if (t->constants) {
8355 efree(t->constants);
8356 t->constants = NULL;
8357 }
8358 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8359 goto exit;
8360 }
8361 memcpy(shared_exit_info, exit_info,
8362 sizeof(zend_jit_trace_exit_info) * t->exit_count);
8363 t->exit_info = shared_exit_info;
8364 }
8365
8366 if (t->stack_map_size) {
8367 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
8368 if (!shared_stack_map) {
8369 efree(t->stack_map);
8370 t->stack_map = NULL;
8371 if (t->constants) {
8372 efree(t->constants);
8373 t->constants = NULL;
8374 }
8375 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8376 goto exit;
8377 }
8378 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
8379 efree(t->stack_map);
8380 t->stack_map = shared_stack_map;
8381 }
8382
8383 if (t->consts_count) {
8384 zend_jit_exit_const *constants = (zend_jit_exit_const*)zend_shared_alloc(t->consts_count * sizeof(zend_jit_exit_const));
8385 if (!constants) {
8386 efree(t->constants);
8387 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8388 goto exit;
8389 }
8390 memcpy(constants, t->constants, t->consts_count * sizeof(zend_jit_exit_const));
8391 efree(t->constants);
8392 t->constants = constants;
8393 }
8394
8395 zend_jit_link_side_trace(
8396 zend_jit_traces[parent_num].code_start,
8397 zend_jit_traces[parent_num].code_size,
8398 zend_jit_traces[parent_num].jmp_table_size,
8399 exit_num,
8400 handler);
8401
8402 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
8403 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
8404
8405 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
8406 ZEND_JIT_TRACE_NUM++;
8407 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
8408
8409 ret = ZEND_JIT_TRACE_STOP_COMPILED;
8410 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
8411 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
8412 if (t->stack_map) {
8413 efree(t->stack_map);
8414 t->stack_map = NULL;
8415 }
8416 if (t->constants) {
8417 efree(t->constants);
8418 t->constants = NULL;
8419 }
8420 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
8421 } else {
8422 if (t->stack_map) {
8423 efree(t->stack_map);
8424 t->stack_map = NULL;
8425 }
8426 if (t->constants) {
8427 efree(t->constants);
8428 t->constants = NULL;
8429 }
8430 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
8431 }
8432
8433 exit:;
8434 } zend_catch {
8435 do_bailout = 1;
8436 } zend_end_try();
8437
8438 zend_jit_protect();
8439 SHM_PROTECT();
8440 }
8441
8442 zend_shared_alloc_unlock();
8443
8444 if (do_bailout) {
8445 zend_bailout();
8446 }
8447
8448 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
8449 && ret == ZEND_JIT_TRACE_STOP_COMPILED
8450 && t->exit_count > 0) {
8451 zend_jit_dump_exit_info(t);
8452 }
8453
8454 return ret;
8455 }
8456
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)8457 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
8458 {
8459 zend_jit_trace_stop stop;
8460 int ret = 0;
8461 uint32_t trace_num;
8462 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8463 uint32_t is_megamorphic = 0;
8464 uint32_t polymorphism = 0;
8465 uint32_t root;
8466 int ret_depth = 0;
8467
8468 trace_num = ZEND_JIT_TRACE_NUM;
8469
8470 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8471 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8472 return 0;
8473 }
8474
8475 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8476 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
8477 trace_num, parent_num, exit_num,
8478 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8479 EX(func)->op_array.scope ? "::" : "",
8480 EX(func)->op_array.function_name ?
8481 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8482 ZSTR_VAL(EX(func)->op_array.filename),
8483 EX(opline)->lineno);
8484 }
8485
8486 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8487 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8488 goto abort;
8489 }
8490
8491 root = zend_jit_traces[parent_num].root;
8492 if (zend_jit_traces[root].child_count >= JIT_G(max_side_traces)) {
8493 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8494 goto abort;
8495 }
8496
8497 if (JIT_G(max_polymorphic_calls) > 0) {
8498 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
8499 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
8500 && EX(call))) {
8501 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
8502 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
8503 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
8504 } else if (!zend_jit_traces[parent_num].polymorphism) {
8505 polymorphism = 1;
8506 } else if (exit_num == 0) {
8507 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
8508 }
8509 }
8510 }
8511
8512 /* Check if this is a side trace of a root LOOP trace */
8513 if ((zend_jit_traces[root].flags & ZEND_JIT_TRACE_LOOP)
8514 && zend_jit_traces[root].op_array != &EX(func)->op_array) {
8515 const zend_op_array *op_array = zend_jit_traces[root].op_array;
8516 const zend_op *opline = zend_jit_traces[root].opline;
8517 zend_jit_op_array_trace_extension *jit_extension =
8518 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8519
8520 if (jit_extension->trace_info[opline - op_array->opcodes].trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8521 zend_execute_data *ex = execute_data;
8522 int n = 0;
8523 do {
8524 ex = ex->prev_execute_data;
8525 n++;
8526 } while (ex && zend_jit_traces[root].op_array != &ex->func->op_array);
8527 if (ex && n <= ZEND_JIT_TRACE_MAX_RET_DEPTH) {
8528 ret_depth = n;
8529 }
8530 }
8531 }
8532
8533 JIT_G(tracing) = 1;
8534 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic, ret_depth);
8535 JIT_G(tracing) = 0;
8536
8537 if (stop & ZEND_JIT_TRACE_HALT) {
8538 ret = -1;
8539 }
8540 stop &= ~ZEND_JIT_TRACE_HALT;
8541
8542 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8543 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8544 const zend_op_array *op_array = trace_buffer[0].op_array;
8545 const zend_op *opline = trace_buffer[1].opline;
8546 zend_jit_op_array_trace_extension *jit_extension =
8547 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8548 size_t offset = jit_extension->offset;
8549
8550 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8551 trace_num,
8552 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8553 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8554 op_array->scope ? "::" : "",
8555 op_array->function_name ?
8556 ZSTR_VAL(op_array->function_name) : "$main",
8557 ZSTR_VAL(op_array->filename),
8558 opline->lineno);
8559 }
8560 }
8561
8562 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8563 zend_jit_dump_trace(trace_buffer, NULL);
8564 }
8565
8566 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8567 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8568 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8569 uint32_t idx = trace_buffer[1].last;
8570 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8571 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8572 trace_num,
8573 link_to);
8574 } else {
8575 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8576 trace_num,
8577 zend_jit_trace_stop_description[stop]);
8578 }
8579 }
8580 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8581 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8582 } else {
8583 const zend_op_array *op_array = trace_buffer[0].op_array;
8584 zend_jit_op_array_trace_extension *jit_extension =
8585 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8586 const zend_op *opline = trace_buffer[1].opline;
8587
8588 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8589 }
8590 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8591 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8592 fprintf(stderr, "---- TRACE %d %s\n",
8593 trace_num,
8594 zend_jit_trace_stop_description[stop]);
8595 }
8596 } else {
8597 goto abort;
8598 }
8599 } else {
8600 abort:
8601 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8602 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8603 trace_num,
8604 zend_jit_trace_stop_description[stop]);
8605 }
8606 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8607 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8608 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8609 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8610 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8611 parent_num, exit_num);
8612 }
8613 }
8614 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8615 execute_data = EG(current_execute_data);
8616 return zend_jit_trace_hot_root(execute_data, EX(opline));
8617 }
8618 }
8619
8620 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8621 fprintf(stderr, "\n");
8622 }
8623
8624 return ret;
8625 }
8626
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8627 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8628 {
8629 uint32_t trace_num = EG(jit_trace_num);
8630 zend_execute_data *execute_data = EG(current_execute_data);
8631 const zend_op *orig_opline = EX(opline);
8632 const zend_op *opline;
8633 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8634 int repeat_last_opline = 0;
8635
8636 /* Deoptimization of VM stack state */
8637 uint32_t i;
8638 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8639 zend_jit_trace_stack *stack = stack_size ? t->stack_map + t->exit_info[exit_num].stack_offset : NULL;
8640
8641 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8642 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8643 call->prev_execute_data = EX(call);
8644 EX(call) = call;
8645 }
8646
8647 for (i = 0; i < stack_size; i++) {
8648 if (STACK_FLAGS(stack, i) == ZREG_CONST) {
8649 if (STACK_TYPE(stack, i) == IS_LONG) {
8650 ZVAL_LONG(EX_VAR_NUM(i), (zend_long)t->constants[STACK_REF(stack, i)].i);
8651 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8652 ZVAL_DOUBLE(EX_VAR_NUM(i), t->constants[STACK_REF(stack, i)].d);
8653 } else {
8654 ZEND_UNREACHABLE();
8655 }
8656 } else if (STACK_FLAGS(stack, i) == ZREG_TYPE_ONLY) {
8657 uint32_t type = STACK_TYPE(stack, i);
8658 if (type <= IS_DOUBLE) {
8659 Z_TYPE_INFO_P(EX_VAR_NUM(i)) = type;
8660 } else {
8661 ZEND_UNREACHABLE();
8662 }
8663 } else if (STACK_FLAGS(stack, i) == ZREG_THIS) {
8664 zend_object *obj = Z_OBJ(EX(This));
8665
8666 GC_ADDREF(obj);
8667 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8668 } else if (STACK_FLAGS(stack, i) == ZREG_ZVAL_ADDREF) {
8669 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8670 } else if (STACK_FLAGS(stack, i) == ZREG_ZVAL_COPY) {
8671 zval *val = (zval*)regs->gpr[STACK_REG(stack, i)];
8672
8673 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8674 /* Undefined array index or property */
8675 const zend_op *op = t->exit_info[exit_num].opline;
8676 ZEND_ASSERT(op);
8677 op--;
8678 if (op->opcode == ZEND_FETCH_DIM_IS || op->opcode == ZEND_FETCH_OBJ_IS) {
8679 ZVAL_NULL(EX_VAR_NUM(i));
8680 } else {
8681 assert(op->opcode == ZEND_FETCH_DIM_R || op->opcode == ZEND_FETCH_LIST_R || op->opcode == ZEND_FETCH_OBJ_R);
8682 repeat_last_opline = 1;
8683 }
8684 } else {
8685 ZVAL_COPY(EX_VAR_NUM(i), val);
8686 }
8687 } else if (STACK_FLAGS(stack, i) & ZREG_SPILL_SLOT) {
8688 ZEND_ASSERT(STACK_REG(stack, i) != ZREG_NONE);
8689 uintptr_t ptr = (uintptr_t)regs->gpr[STACK_REG(stack, i)] + STACK_REF(stack, i);
8690
8691 if (STACK_TYPE(stack, i) == IS_LONG) {
8692 ZVAL_LONG(EX_VAR_NUM(i), *(zend_long*)ptr);
8693 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8694 ZVAL_DOUBLE(EX_VAR_NUM(i), *(double*)ptr);
8695 } else {
8696 ZEND_UNREACHABLE();
8697 }
8698 } else if (STACK_REG(stack, i) != ZREG_NONE) {
8699 if (STACK_TYPE(stack, i) == IS_LONG) {
8700 zend_long val = regs->gpr[STACK_REG(stack, i)];
8701 ZVAL_LONG(EX_VAR_NUM(i), val);
8702 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8703 double val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8704 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8705 } else {
8706 ZEND_UNREACHABLE();
8707 }
8708 }
8709 }
8710
8711 if (repeat_last_opline) {
8712 EX(opline) = t->exit_info[exit_num].opline - 1;
8713 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8714 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8715 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8716 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8717 }
8718 return 1;
8719 }
8720
8721 opline = t->exit_info[exit_num].opline;
8722
8723 if (opline) {
8724 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8725 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8726 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8727 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8728 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8729 EX(opline) = opline-1;
8730 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8731 }
8732 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8733 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8734 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8735 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8736 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8737 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8738 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8739 EX(opline) = opline-1;
8740 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8741 }
8742 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8743 if (EG(exception)) {
8744 return 1;
8745 }
8746 }
8747 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8748 ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0);
8749 zend_function *func = (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg];
8750
8751 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8752 zend_string_release_ex(func->common.function_name, 0);
8753 zend_free_trampoline(func);
8754 EX(opline) = opline;
8755 return 1;
8756 }
8757 }
8758
8759 /* Set VM opline to continue interpretation */
8760 EX(opline) = opline;
8761 }
8762
8763 if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) {
8764 return 1;
8765 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8766 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8767 return 0;
8768 }
8769
8770 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8771 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8772 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8773
8774 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8775 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8776 trace_num,
8777 exit_num,
8778 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8779 EX(func)->op_array.scope ? "::" : "",
8780 EX(func)->op_array.function_name ?
8781 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8782 ZSTR_VAL(EX(func)->op_array.filename),
8783 EX(opline)->lineno);
8784 }
8785
8786 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8787 zend_jit_op_array_trace_extension *jit_extension;
8788 uint32_t num = trace_num;
8789
8790 while (t->root != num) {
8791 num = t->root;
8792 t = &zend_jit_traces[num];
8793 }
8794
8795 zend_shared_alloc_lock();
8796
8797 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8798
8799 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8800 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8801 /* skip: not JIT-ed nor blacklisted */
8802 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8803 /* too many root traces, blacklist the root trace */
8804 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
8805 SHM_UNPROTECT();
8806 zend_jit_unprotect();
8807
8808 ((zend_op*)opline)->handler =
8809 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->orig_handler;
8810
8811 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= ~ZEND_JIT_TRACE_JITED;
8812 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
8813
8814 zend_jit_protect();
8815 SHM_PROTECT();
8816 }
8817 } else {
8818 SHM_UNPROTECT();
8819 zend_jit_unprotect();
8820
8821 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8822 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8823 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8824 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8825 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8826 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8827 }
8828 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8829 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8830
8831 zend_jit_protect();
8832 SHM_PROTECT();
8833 }
8834
8835 zend_shared_alloc_unlock();
8836
8837 return 0;
8838 }
8839
8840 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8841 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8842 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8843 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8844 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8845 trace_num, exit_num);
8846 }
8847 return 0;
8848 }
8849 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8850 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8851 }
8852
8853 /* Return 1 to call original handler instead of the same JIT-ed trace */
8854 return (orig_opline == t->opline && EX(opline) == orig_opline);
8855 }
8856
zend_jit_trace_supported(const zend_op * opline)8857 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8858 {
8859 switch (opline->opcode) {
8860 case ZEND_CATCH:
8861 case ZEND_FAST_CALL:
8862 case ZEND_FAST_RET:
8863 return ZEND_JIT_TRACE_UNSUPPORTED;
8864 default:
8865 return ZEND_JIT_TRACE_SUPPORTED;
8866 }
8867 }
8868
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8869 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8870 {
8871 zend_jit_op_array_trace_extension *jit_extension;
8872 uint32_t i;
8873
8874 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8875 for (i = 0; i < op_array->last; i++) {
8876 jit_extension->trace_info[i].trace_flags &=
8877 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8878 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8879 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8880 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8881 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8882 } else {
8883 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8884 }
8885 }
8886 return SUCCESS;
8887 }
8888
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8889 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8890 {
8891 zend_op *opline;
8892 zend_jit_op_array_trace_extension *jit_extension;
8893 uint32_t i;
8894
8895 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8896
8897 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8898 if (!jit_extension) {
8899 return FAILURE;
8900 }
8901 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8902 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8903 jit_extension->op_array = op_array;
8904 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8905 for (i = 0; i < op_array->last; i++) {
8906 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8907 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8908 jit_extension->trace_info[i].counter = NULL;
8909 jit_extension->trace_info[i].trace_flags =
8910 zend_jit_trace_supported(&op_array->opcodes[i]);
8911 }
8912 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8913
8914 if (JIT_G(hot_loop)) {
8915 zend_cfg cfg;
8916
8917 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8918
8919 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8920 return FAILURE;
8921 }
8922
8923 for (i = 0; i < cfg.blocks_count; i++) {
8924 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8925 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8926 /* loop header */
8927 opline = op_array->opcodes + cfg.blocks[i].start;
8928 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8929 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8930 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8931 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8932 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8933 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8934 }
8935 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8936 ZEND_JIT_TRACE_START_LOOP;
8937 }
8938 }
8939 }
8940 }
8941 }
8942
8943 if (JIT_G(hot_func)) {
8944 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8945 opline = op_array->opcodes;
8946 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8947 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8948 opline++;
8949 }
8950 }
8951
8952 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8953 /* function entry */
8954 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8955 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8956 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8957 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8958 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8959 ZEND_JIT_TRACE_START_ENTER;
8960 }
8961 }
8962
8963 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8964
8965 return SUCCESS;
8966 }
8967
zend_jit_trace_init_caches(void)8968 static void zend_jit_trace_init_caches(void)
8969 {
8970 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8971 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8972 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8973 JIT_G(bad_root_slot) = 0;
8974
8975 if (JIT_G(exit_counters)) {
8976 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8977 }
8978 }
8979
zend_jit_trace_reset_caches(void)8980 static void zend_jit_trace_reset_caches(void)
8981 {
8982 JIT_G(tracing) = 0;
8983 #ifdef ZTS
8984 if (!JIT_G(exit_counters)) {
8985 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8986 }
8987 #endif
8988 }
8989
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8990 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8991 {
8992 if (jit_globals->exit_counters) {
8993 free(jit_globals->exit_counters);
8994 }
8995 }
8996
zend_jit_trace_restart(void)8997 static void zend_jit_trace_restart(void)
8998 {
8999 ZEND_JIT_TRACE_NUM = 1;
9000 ZEND_JIT_COUNTER_NUM = 0;
9001 ZEND_JIT_EXIT_NUM = 0;
9002 ZEND_JIT_EXIT_COUNTERS = 0;
9003 ZCSG(jit_counters_stopped) = false;
9004
9005 zend_jit_trace_init_caches();
9006 }
9007
