1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 #define USE_ABSTRACT_STACK_FOR_RES_USE_INFO 1
32
33 static const char * zend_jit_trace_stop_description[] = {
34 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
35 };
36
zend_jit_trace_star_desc(uint8_t trace_flags)37 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
38 {
39 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
40 return "loop";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
42 return "enter";
43 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
44 return "return";
45 } else {
46 ZEND_UNREACHABLE();
47 return "???";
48 }
49 }
50
zend_jit_trace_startup(zend_bool reattached)51 static int zend_jit_trace_startup(zend_bool reattached)
52 {
53 if (!reattached) {
54 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
55 if (!zend_jit_traces) {
56 return FAILURE;
57 }
58 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
59 if (!zend_jit_exit_groups) {
60 return FAILURE;
61 }
62 ZEND_JIT_TRACE_NUM = 1;
63 ZEND_JIT_COUNTER_NUM = 0;
64 ZEND_JIT_EXIT_NUM = 0;
65 ZEND_JIT_EXIT_COUNTERS = 0;
66 ZCSG(jit_traces) = zend_jit_traces;
67 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
68 } else {
69 zend_jit_traces = ZCSG(jit_traces);
70 if (!zend_jit_traces) {
71 return FAILURE;
72 }
73 zend_jit_exit_groups = ZCSG(jit_exit_groups);
74 if (!zend_jit_exit_groups) {
75 return FAILURE;
76 }
77 }
78
79 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
80 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
81
82 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
83 if (JIT_G(exit_counters) == NULL) {
84 return FAILURE;
85 }
86
87 return SUCCESS;
88 }
89
zend_jit_trace_allocate_exit_group(uint32_t n)90 static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
91 {
92 dasm_State* dasm_state = NULL;
93 const void *entry;
94 char name[32];
95
96 dasm_init(&dasm_state, DASM_MAXSECTION);
97 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
98 dasm_setup(&dasm_state, dasm_actions);
99 zend_jit_trace_exit_group_stub(&dasm_state, n);
100
101 sprintf(name, "jit$$trace_exit_%d", n);
102 entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0);
103 dasm_free(&dasm_state);
104
105 #ifdef HAVE_DISASM
106 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
107 uint32_t i;
108
109 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
110 sprintf(name, "jit$$trace_exit_%d", n + i);
111 zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
112 }
113 }
114 #endif
115
116 return entry;
117 }
118
zend_jit_trace_allocate_exit_point(uint32_t n)119 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
120 {
121 const void *group = NULL;
122
123 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
124 return NULL;
125 }
126 do {
127 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
128 if (!group) {
129 return NULL;
130 }
131 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
132 group;
133 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
134 } while (n >= ZEND_JIT_EXIT_NUM);
135 return (const void*)
136 ((const char*)group +
137 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
138 }
139
zend_jit_trace_get_exit_addr(uint32_t n)140 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
141 {
142 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
143 return zend_jit_trace_allocate_exit_point(n);
144 }
145 return (const void*)
146 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
147 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
148 }
149
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)150 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
151 {
152 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
153 uint32_t exit_point;
154 const zend_op_array *op_array;
155 uint32_t stack_offset = (uint32_t)-1;
156 uint32_t stack_size;
157 zend_jit_trace_stack *stack = NULL;
158
159 if (delayed_call_chain) {
160 assert(to_opline != NULL); /* CALL and IP share the same register */
161 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
162 }
163 if (JIT_G(current_frame)) {
164 op_array = &JIT_G(current_frame)->func->op_array;
165 stack_size = op_array->last_var + op_array->T;
166 if (stack_size) {
167 stack = JIT_G(current_frame)->stack;
168 do {
169 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
170 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
171 || STACK_REG(stack, stack_size-1) != ZREG_NONE) {
172 break;
173 }
174 stack_size--;
175 } while (stack_size);
176 }
177 } else {
178 op_array = NULL;
179 stack_size = 0;
180 }
181
182 /* Try to reuse exit points */
183 if (to_opline != NULL && t->exit_count > 0) {
184 uint32_t i = t->exit_count;
185
186 do {
187 i--;
188 if (stack_size == 0
189 || (t->exit_info[i].stack_size >= stack_size
190 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
191 stack_offset = t->exit_info[i].stack_offset;
192 if (t->exit_info[i].opline == to_opline
193 && t->exit_info[i].flags == flags
194 && t->exit_info[i].stack_size == stack_size) {
195 return i;
196 }
197 }
198 } while (i > 0);
199 }
200
201 exit_point = t->exit_count;
202 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
203 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
204 stack_offset = t->stack_map_size;
205 t->stack_map_size += stack_size;
206 // TODO: reduce number of reallocations ???
207 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
208 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
209 }
210 t->exit_count++;
211 t->exit_info[exit_point].opline = to_opline;
212 t->exit_info[exit_point].op_array = op_array;
213 t->exit_info[exit_point].flags = flags;
214 t->exit_info[exit_point].stack_size = stack_size;
215 t->exit_info[exit_point].stack_offset = stack_offset;
216 }
217
218 return exit_point;
219 }
220
zend_jit_trace_add_code(const void * start,uint32_t size)221 static void zend_jit_trace_add_code(const void *start, uint32_t size)
222 {
223 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
224
225 t->code_start = start;
226 t->code_size = size;
227 }
228
zend_jit_find_trace(const void * addr)229 static uint32_t zend_jit_find_trace(const void *addr)
230 {
231 uint32_t i;
232
233 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
234 if (zend_jit_traces[i].code_start == addr) {
235 return i;
236 }
237 }
238 ZEND_UNREACHABLE();
239 return 0;
240 }
241
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)242 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
243 {
244 smart_str buf = {0};
245
246 smart_str_appends(&buf, TRACE_PREFIX);
247 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
248 smart_str_appendc(&buf, '$');
249 if (op_array->function_name) {
250 if (op_array->scope) {
251 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
252 smart_str_appends(&buf, "::");
253 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
254 } else {
255 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
256 }
257 } else if (op_array->filename) {
258 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
259 }
260 smart_str_appendc(&buf, '$');
261 smart_str_append_long(&buf, (zend_long)lineno);
262 smart_str_0(&buf);
263 return buf.s;
264 }
265
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)266 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
267 {
268 switch (opline->opcode) {
269 case ZEND_IS_IDENTICAL:
270 case ZEND_IS_NOT_IDENTICAL:
271 case ZEND_IS_EQUAL:
272 case ZEND_IS_NOT_EQUAL:
273 case ZEND_IS_SMALLER:
274 case ZEND_IS_SMALLER_OR_EQUAL:
275 case ZEND_CASE:
276 case ZEND_CASE_STRICT:
277 case ZEND_ISSET_ISEMPTY_CV:
278 case ZEND_ISSET_ISEMPTY_VAR:
279 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
280 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
281 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
282 case ZEND_INSTANCEOF:
283 case ZEND_TYPE_CHECK:
284 case ZEND_DEFINED:
285 case ZEND_IN_ARRAY:
286 case ZEND_ARRAY_KEY_EXISTS:
287 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
288 /* smart branch */
289 return 1;
290 }
291 break;
292 case ZEND_JMPZNZ:
293 case ZEND_JMPZ:
294 case ZEND_JMPNZ:
295 case ZEND_JMPZ_EX:
296 case ZEND_JMPNZ_EX:
297 case ZEND_JMP_SET:
298 case ZEND_COALESCE:
299 case ZEND_JMP_NULL:
300 case ZEND_FE_RESET_R:
301 case ZEND_FE_RESET_RW:
302 case ZEND_ASSERT_CHECK:
303 case ZEND_FE_FETCH_R:
304 case ZEND_FE_FETCH_RW:
305 case ZEND_SWITCH_LONG:
306 case ZEND_SWITCH_STRING:
307 case ZEND_MATCH:
308 /* branch opcodes */
309 return 1;
310 case ZEND_NEW:
311 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
312 /* NEW may skip constructor without arguments */
313 return 1;
314 }
315 break;
316 case ZEND_CATCH:
317 case ZEND_FAST_CALL:
318 case ZEND_FAST_RET:
319 case ZEND_GENERATOR_CREATE:
320 case ZEND_GENERATOR_RETURN:
321 case ZEND_EXIT:
322 case ZEND_YIELD:
323 case ZEND_YIELD_FROM:
324 case ZEND_INCLUDE_OR_EVAL:
325 case ZEND_MATCH_ERROR:
326 /* unsupported */
327 return 1;
328 case ZEND_DO_FCALL:
329 /* potentially polymorphic call */
330 return 1;
331 #if 0
332 case ZEND_DO_UCALL:
333 case ZEND_DO_FCALL_BY_NAME:
334 /* monomorphic call */
335 // TODO: recompilation may change target ???
336 return 0;
337 #endif
338 case ZEND_RETURN_BY_REF:
339 case ZEND_RETURN:
340 /* return */
341 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
342 default:
343 break;
344 }
345 return 0;
346 }
347
zend_jit_trace_type_to_info_ex(zend_uchar type,uint32_t info)348 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(zend_uchar type, uint32_t info)
349 {
350 if (type == IS_UNKNOWN) {
351 return info;
352 }
353 ZEND_ASSERT(info & (1 << type));
354 if (type < IS_STRING) {
355 return (1 << type);
356 } else if (type != IS_ARRAY) {
357 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
358 } else {
359 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
360 }
361 }
362
zend_jit_trace_type_to_info(zend_uchar type)363 static zend_always_inline uint32_t zend_jit_trace_type_to_info(zend_uchar type)
364 {
365 return zend_jit_trace_type_to_info_ex(type, -1);
366 }
367
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)368 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
369 {
370 if (var >= op_array->last_var) {
371 return NO_ALIAS;
372 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
373 return SYMTABLE_ALIAS;
374 } else if (ssa->vars) {
375 return ssa->vars[var].alias;
376 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
377 return HTTP_RESPONSE_HEADER_ALIAS;
378 }
379 return NO_ALIAS;
380 }
381
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)382 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
383 int ssa_var,
384 uint8_t op_type)
385 {
386 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
387
388 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
389 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
390 info->type |= MAY_BE_GUARD;
391 } else {
392 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
393 }
394 }
395 }
396
397 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
398 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
399 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
400 } \
401 } while (0)
402
403 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
404 if (op_type != IS_UNKNOWN) { \
405 if ((op_info & MAY_BE_GUARD) != 0) { \
406 if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \
407 goto jit_failure; \
408 } \
409 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
410 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
411 op_info = zend_jit_trace_type_to_info(op_type); \
412 } else { \
413 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
414 op_info &= ~MAY_BE_GUARD; \
415 ssa->var_info[_ssa_var].type &= op_info; \
416 } \
417 } \
418 } \
419 } while (0)
420
421 #define ADD_OP1_TRACE_GUARD() \
422 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
423 #define ADD_OP2_TRACE_GUARD() \
424 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
425 #define ADD_OP1_DATA_TRACE_GUARD() \
426 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
427
428 #define CHECK_OP1_TRACE_TYPE() \
429 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
430 #define CHECK_OP2_TRACE_TYPE() \
431 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
432 #define CHECK_OP1_DATA_TRACE_TYPE() \
433 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
434
zend_jit_trace_frame_size(const zend_op_array * op_array)435 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
436 {
437 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
438 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
439 } else {
440 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
441 }
442 }
443
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)444 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
445 {
446 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
447 }
448
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)449 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
450 {
451 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
452 }
453
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,zend_uchar type)454 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, zend_uchar type)
455 {
456 zend_jit_trace_stack *stack = call->stack;
457 const zend_op_array *op_array = &call->func->op_array;
458 uint32_t arg_num = opline->op2.num;
459
460 if (op_array->type != ZEND_USER_FUNCTION
461 || arg_num > op_array->num_args) {
462 return;
463 }
464 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
465 zend_arg_info *arg_info;
466
467 ZEND_ASSERT(arg_num <= op_array->num_args);
468 arg_info = &op_array->arg_info[arg_num-1];
469
470 if (ZEND_TYPE_IS_SET(arg_info->type)) {
471 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
472 return;
473 }
474 }
475 }
476 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
477 }
478
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)479 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
480 {
481 zend_jit_op_array_trace_extension *jit_extension;
482 zend_ssa *ssa;
483
484 jit_extension =
485 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
486 jit_extension->func_info.num = 0;
487 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
488 | ZEND_FUNC_JIT_ON_PROF_REQUEST
489 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
490 | ZEND_FUNC_JIT_ON_HOT_TRACE;
491 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
492 ssa = &jit_extension->func_info.ssa;
493
494 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
495 do {
496 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
497 break;
498 }
499
500 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
501 if (zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info) != SUCCESS) {
502 break;
503 }
504 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
505 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
506 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
507 }
508 }
509
510 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
511 break;
512 }
513
514 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
515 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
516 }
517 return ssa;
518 } while (0);
519 }
520
521 memset(ssa, 0, sizeof(zend_ssa));
522 ssa->cfg.blocks_count = 1;
523 return ssa;
524 }
525
526 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
527 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
528
zend_jit_trace_op_len(const zend_op * opline)529 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
530 {
531 int len;
532
533 switch (opline->opcode) {
534 case ZEND_ASSIGN_DIM:
535 case ZEND_ASSIGN_OBJ:
536 case ZEND_ASSIGN_STATIC_PROP:
537 case ZEND_ASSIGN_DIM_OP:
538 case ZEND_ASSIGN_OBJ_OP:
539 case ZEND_ASSIGN_STATIC_PROP_OP:
540 case ZEND_ASSIGN_OBJ_REF:
541 case ZEND_ASSIGN_STATIC_PROP_REF:
542 return 2; /* OP_DATA */
543 case ZEND_RECV_INIT:
544 len = 1;
545 opline++;
546 while (opline->opcode == ZEND_RECV_INIT) {
547 len++;
548 opline++;
549 }
550 return len;
551 case ZEND_BIND_GLOBAL:
552 len = 1;
553 opline++;
554 while (opline->opcode == ZEND_BIND_GLOBAL) {
555 len++;
556 opline++;
557 }
558 return len;
559 // case ZEND_IS_IDENTICAL:
560 // case ZEND_IS_NOT_IDENTICAL:
561 // case ZEND_IS_EQUAL:
562 // case ZEND_IS_NOT_EQUAL:
563 // case ZEND_IS_SMALLER:
564 // case ZEND_IS_SMALLER_OR_EQUAL:
565 // case ZEND_CASE:
566 // case ZEND_ISSET_ISEMPTY_CV:
567 // case ZEND_ISSET_ISEMPTY_VAR:
568 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
569 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
570 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
571 // case ZEND_INSTANCEOF:
572 // case ZEND_TYPE_CHECK:
573 // case ZEND_DEFINED:
574 // case ZEND_IN_ARRAY:
575 // case ZEND_ARRAY_KEY_EXISTS:
576 default:
577 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
578 return 2; /* JMPZ/JMPNZ */
579 }
580 return 1;
581 }
582 }
583
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)584 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
585 {
586 const zend_op_array *op_array;
587 zend_jit_trace_rec *p;
588 int k, vars_count;
589 zend_bitset use, def;
590 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
591 uint32_t set_size;
592 zend_ssa_phi *prev = NULL;
593 int level = 0;
594 ALLOCA_FLAG(use_heap);
595
596 op_array = trace_buffer->op_array;
597 set_size = zend_bitset_len(op_array->last_var + op_array->T);
598 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
599 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
600 def = use + set_size;
601 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
602 for (;;p++) {
603 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
604 const zend_op *opline = p->opline;
605 int len;
606
607 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
608 len = zend_jit_trace_op_len(opline);
609 while (len > 1) {
610 opline++;
611 if (opline->opcode != ZEND_OP_DATA) {
612 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
613 }
614 len--;
615 }
616 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
617 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
618 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
619 level++;
620 } else if (p->op == ZEND_JIT_TRACE_BACK) {
621 if (level == 0) {
622 // Phi for recursive calls and returns are not supported yet ???
623 assert(0);
624 } else {
625 level--;
626 }
627 } else if (p->op == ZEND_JIT_TRACE_END) {
628 break;
629 }
630 }
631
632 zend_bitset_intersection(use, def, set_size);
633
634 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
635 vars_count = op_array->last_var;
636 } else {
637 vars_count = op_array->last_var + op_array->T;
638 }
639 for (k = 0; k < vars_count; k++) {
640 if (zend_bitset_in(use, k)) {
641 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
642 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
643 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
644 sizeof(void*) * 2);
645 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
646 phi->sources[0] = STACK_VAR(stack, k);
647 phi->sources[1] = -1;
648 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
649 phi->pi = -1;
650 phi->var = k;
651 phi->ssa_var = ssa_vars_count;
652 SET_STACK_VAR(stack, k, ssa_vars_count);
653 ssa_vars_count++;
654 phi->block = 1;
655 if (prev) {
656 prev->next = phi;
657 } else {
658 tssa->blocks[1].phis = phi;
659 }
660 prev = phi;
661 }
662 }
663
664 free_alloca(use, use_heap);
665
666 return ssa_vars_count;
667 }
668
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)669 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
670 {
671 zend_ssa_phi *prev = NULL;
672 const zend_op_array *op_array = trace_buffer->op_array;
673 const zend_op *opline = trace_buffer[1].opline;
674 int count = opline - op_array->opcodes;
675 int i;
676
677 for(i = 0; i < count; i++) {
678 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
679 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
680 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
681 sizeof(void*) * 2);
682 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
683 phi->sources[0] = STACK_VAR(stack, i);
684 phi->sources[1] = -1;
685 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
686 phi->pi = -1;
687 phi->var = i;
688 phi->ssa_var = ssa_vars_count;
689 SET_STACK_VAR(stack, i, ssa_vars_count);
690 ssa_vars_count++;
691 phi->block = 1;
692 if (prev) {
693 prev->next = phi;
694 } else {
695 tssa->blocks[1].phis = phi;
696 }
697 prev = phi;
698 }
699 return ssa_vars_count;
700 }
701
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)702 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
703 {
704 const zend_op *opline = trace_buffer[1].opline - 1;
705 int i;
706
707 if (RETURN_VALUE_USED(opline)) {
708 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
709 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
710 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
711 sizeof(void*) * 2);
712
713 i = EX_VAR_TO_NUM(opline->result.var);
714 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
715 phi->sources[0] = STACK_VAR(stack, i);
716 phi->sources[1] = -1;
717 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
718 phi->pi = -1;
719 phi->var = i;
720 phi->ssa_var = ssa_vars_count;
721 SET_STACK_VAR(stack, i, ssa_vars_count);
722 ssa_vars_count++;
723 phi->block = 1;
724 tssa->blocks[1].phis = phi;
725 }
726 return ssa_vars_count;
727 }
728
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)729 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
730 {
731 int var, use;
732 zend_ssa_op *op;
733 zend_ssa_var_info *info;
734 unsigned int no_val;
735 zend_ssa_alias_kind alias;
736
737 if (tssa->vars[ssa_var].phi_use_chain) {
738 // TODO: this may be incorrect ???
739 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
740 } else {
741 var = ssa_var;
742 }
743 use = tssa->vars[var].use_chain;
744 if (use >= 0) {
745 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
746 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
747 if (tssa->ops[use].op1_use == var) {
748 no_val = ssa->vars[op->op1_use].no_val;
749 alias = ssa->vars[op->op1_use].alias;
750 info = ssa->var_info + op->op1_use;
751 } else if (tssa->ops[use].op2_use == var) {
752 no_val = ssa->vars[op->op2_use].no_val;
753 alias = ssa->vars[op->op2_use].alias;
754 info = ssa->var_info + op->op2_use;
755 } else if (tssa->ops[use].result_use == var) {
756 no_val = ssa->vars[op->result_use].no_val;
757 alias = ssa->vars[op->result_use].alias;
758 info = ssa->var_info + op->result_use;
759 } else {
760 assert(0);
761 return 0;
762 }
763 tssa->vars[ssa_var].no_val = no_val;
764 tssa->vars[ssa_var].alias = alias;
765 memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info));
766 return 1;
767 }
768 return 0;
769 }
770
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)771 static int zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
772 {
773 int def;
774 zend_ssa_op *op;
775 zend_ssa_var_info *info;
776 unsigned int no_val;
777 zend_ssa_alias_kind alias;
778
779 def = tssa->vars[ssa_var].definition;
780 if (def >= 0) {
781 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
782 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
783 if (tssa->ops[def].op1_def == ssa_var) {
784 no_val = ssa->vars[op->op1_def].no_val;
785 alias = ssa->vars[op->op1_def].alias;
786 info = ssa->var_info + op->op1_def;
787 } else if (tssa->ops[def].op2_def == ssa_var) {
788 no_val = ssa->vars[op->op2_def].no_val;
789 alias = ssa->vars[op->op2_def].alias;
790 info = ssa->var_info + op->op2_def;
791 } else if (tssa->ops[def].result_def == ssa_var) {
792 no_val = ssa->vars[op->result_def].no_val;
793 alias = ssa->vars[op->result_def].alias;
794 info = ssa->var_info + op->result_def;
795 } else {
796 assert(0);
797 return 0;
798 }
799
800 tssa->vars[ssa_var].no_val = no_val;
801 tssa->vars[ssa_var].alias = alias;
802
803 if (info->has_range) {
804 if (tssa->var_info[ssa_var].has_range) {
805 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
806 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
807 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
808 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
809 } else {
810 tssa->var_info[ssa_var].has_range = 1;
811 tssa->var_info[ssa_var].range = info->range;
812 }
813 }
814 return 1;
815 }
816 return 0;
817 }
818
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)819 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
820 {
821 int def;
822 zend_ssa_op *op;
823 zend_ssa_var_info *info;
824
825 def = tssa->vars[ssa_var].definition;
826 if (def >= 0) {
827 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
828 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
829 if (tssa->ops[def].op1_def == ssa_var) {
830 info = ssa->var_info + op->op1_def;
831 } else if (tssa->ops[def].op2_def == ssa_var) {
832 info = ssa->var_info + op->op2_def;
833 } else if (tssa->ops[def].result_def == ssa_var) {
834 info = ssa->var_info + op->result_def;
835 } else {
836 assert(0);
837 return 0;
838 }
839 tssa->var_info[ssa_var].type &= info->type;
840 if (info->ce) {
841 if (tssa->var_info[ssa_var].ce) {
842 if (tssa->var_info[ssa_var].ce != info->ce) {
843 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
844 /* everything fine */
845 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
846 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
847 } else {
848 // TODO: classes may implement the same interface ???
849 //ZEND_UNREACHABLE();
850 }
851 }
852 tssa->var_info[ssa_var].is_instanceof =
853 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
854 } else {
855 tssa->var_info[ssa_var].ce = info->ce;
856 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
857 }
858 }
859 if (info->has_range) {
860 if (tssa->var_info[ssa_var].has_range) {
861 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
862 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
863 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
864 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
865 } else {
866 tssa->var_info[ssa_var].has_range = 1;
867 tssa->var_info[ssa_var].range = info->range;
868 }
869 }
870 return 1;
871 }
872 return 0;
873 }
874
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)875 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
876 {
877 while (1) {
878 if (p->op == ZEND_JIT_TRACE_VM) {
879 if (p->opline->opcode == ZEND_DO_UCALL
880 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
881 || p->opline->opcode == ZEND_DO_FCALL) {
882 if (p->opline->result_type != IS_UNUSED) {
883 return ssa_op->result_def;
884 }
885 }
886 return -1;
887 } else if (p->op == ZEND_JIT_TRACE_OP1_TYPE || p->op == ZEND_JIT_TRACE_OP2_TYPE) {
888 /*skip */
889 } else {
890 return -1;
891 }
892 p--;
893 }
894 }
895
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)896 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
897 {
898 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
899 p--;
900 while (1) {
901 if (p->op == ZEND_JIT_TRACE_VM) {
902 if (p->opline->opcode == ZEND_INIT_FCALL
903 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
904 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
905 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
906 || p->opline->opcode == ZEND_INIT_USER_CALL
907 || p->opline->opcode == ZEND_NEW
908 || p->opline->opcode == ZEND_INIT_METHOD_CALL
909 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
910 return p->opline;
911 }
912 return NULL;
913 } else if (p->op == ZEND_JIT_TRACE_OP1_TYPE || p->op == ZEND_JIT_TRACE_OP2_TYPE) {
914 /*skip */
915 } else {
916 return NULL;
917 }
918 p--;
919 }
920 } else {
921 const zend_op *opline = NULL;
922 int call_level = 0;
923
924 p++;
925 while (1) {
926 if (p->op == ZEND_JIT_TRACE_VM) {
927 opline = p->opline;
928 break;
929 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
930 call_level++;
931 /*skip */
932 } else {
933 return NULL;
934 }
935 p--;
936 }
937 if (opline) {
938 while (opline > op_array->opcodes) {
939 opline--;
940 switch (opline->opcode) {
941 case ZEND_INIT_FCALL:
942 case ZEND_INIT_FCALL_BY_NAME:
943 case ZEND_INIT_NS_FCALL_BY_NAME:
944 case ZEND_INIT_METHOD_CALL:
945 case ZEND_INIT_DYNAMIC_CALL:
946 case ZEND_INIT_STATIC_METHOD_CALL:
947 case ZEND_INIT_USER_CALL:
948 case ZEND_NEW:
949 if (call_level == 0) {
950 return opline;
951 }
952 call_level--;
953 break;
954 case ZEND_DO_FCALL:
955 case ZEND_DO_ICALL:
956 case ZEND_DO_UCALL:
957 case ZEND_DO_FCALL_BY_NAME:
958 call_level++;
959 break;
960 }
961 }
962 }
963 }
964 return NULL;
965 }
966
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)967 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
968 {
969 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
970 && !(tssa->var_info[var].type & MAY_BE_REF)) {
971 int idx = tssa->vars[var].definition;
972
973 if (idx >= 0) {
974 if (tssa->ops[idx].op1_def == var) {
975 const zend_op *opline = ssa_opcodes[idx];
976 if (opline->opcode == ZEND_PRE_DEC
977 || opline->opcode == ZEND_PRE_INC
978 || opline->opcode == ZEND_POST_DEC
979 || opline->opcode == ZEND_POST_INC) {
980 if (tssa->ops[idx].op1_use >= 0
981 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
982 return 0;
983 }
984 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
985 return 0;
986 }
987 return 1;
988 } else if (opline->opcode == ZEND_ASSIGN_OP
989 && (opline->extended_value == ZEND_ADD
990 || opline->extended_value == ZEND_SUB
991 || opline->extended_value == ZEND_MUL)) {
992 if ((opline->op2_type & (IS_VAR|IS_CV))
993 && tssa->ops[idx].op2_use >= 0
994 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
995 return 0;
996 }
997 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
998 return 0;
999 }
1000 if (opline->op2_type == IS_CONST) {
1001 zval *zv = RT_CONSTANT(opline, opline->op2);
1002 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1003 return 0;
1004 }
1005 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1006 return 0;
1007 }
1008 return 1;
1009 }
1010 }
1011 if (tssa->ops[idx].result_def == var) {
1012 const zend_op *opline = ssa_opcodes[idx];
1013 if (opline->opcode == ZEND_ADD
1014 || opline->opcode == ZEND_SUB
1015 || opline->opcode == ZEND_MUL) {
1016 if ((opline->op1_type & (IS_VAR|IS_CV))
1017 && tssa->ops[idx].op1_use >= 0
1018 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1019 return 0;
1020 }
1021 if ((opline->op2_type & (IS_VAR|IS_CV))
1022 && tssa->ops[idx].op2_use >= 0
1023 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1024 return 0;
1025 }
1026 if (opline->op1_type == IS_CONST) {
1027 zval *zv = RT_CONSTANT(opline, opline->op1);
1028 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1029 return 0;
1030 }
1031 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1032 return 0;
1033 }
1034 if (opline->op2_type == IS_CONST) {
1035 zval *zv = RT_CONSTANT(opline, opline->op2);
1036 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1037 return 0;
1038 }
1039 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1040 return 0;
1041 }
1042 } else if (opline->opcode == ZEND_PRE_DEC
1043 || opline->opcode == ZEND_PRE_INC
1044 || opline->opcode == ZEND_POST_DEC
1045 || opline->opcode == ZEND_POST_INC) {
1046 if ((opline->op1_type & (IS_VAR|IS_CV))
1047 && tssa->ops[idx].op1_use >= 0
1048 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1049 return 0;
1050 }
1051 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1052 return 0;
1053 }
1054 return 1;
1055 }
1056 }
1057 }
1058 }
1059 return 0;
1060 }
1061
1062 typedef struct _zend_tssa {
1063 zend_ssa ssa;
1064 const zend_op **tssa_opcodes;
1065 int used_stack;
1066 } zend_tssa;
1067
1068 static const zend_op _nop_opcode = {0};
1069
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1070 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1071 {
1072 zend_ssa *tssa;
1073 zend_ssa_op *ssa_ops, *op;
1074 zend_ssa_var *ssa_vars;
1075 zend_ssa_var_info *ssa_var_info;
1076 const zend_op_array *op_array;
1077 const zend_op *opline;
1078 const zend_op **ssa_opcodes;
1079 zend_jit_trace_rec *p;
1080 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1081 zend_jit_trace_stack *stack;
1082 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1083 uint32_t optimization_level = 0;
1084 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1085 size_t frame_size, stack_top, stack_size, stack_bottom;
1086 zend_jit_op_array_trace_extension *jit_extension;
1087 zend_ssa *ssa;
1088 zend_jit_trace_stack_frame *frame, *top, *call;
1089 zend_ssa_var_info return_value_info;
1090
1091 /* 1. Count number of TSSA opcodes;
1092 * Count number of activation frames;
1093 * Calculate size of abstract stack;
1094 * Construct regular SSA for involved op_array */
1095 op_array = trace_buffer->op_array;
1096 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1097 stack_bottom = 0;
1098 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1099 ssa_ops_count = 0;
1100 call_level = 0;
1101 level = 0;
1102 num_op_arrays = 0;
1103 /* Remember op_array to cleanup */
1104 op_arrays[num_op_arrays++] = op_array;
1105 /* Build SSA */
1106 ssa = zend_jit_trace_build_ssa(op_array, script);
1107 for (;;p++) {
1108 if (p->op == ZEND_JIT_TRACE_VM) {
1109 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1110 const zend_op *opline = p->opline;
1111
1112 switch (opline->opcode) {
1113 case ZEND_INCLUDE_OR_EVAL:
1114 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1115 break;
1116 case ZEND_FETCH_R:
1117 case ZEND_FETCH_W:
1118 case ZEND_FETCH_RW:
1119 case ZEND_FETCH_FUNC_ARG:
1120 case ZEND_FETCH_IS:
1121 case ZEND_FETCH_UNSET:
1122 case ZEND_UNSET_VAR:
1123 case ZEND_ISSET_ISEMPTY_VAR:
1124 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1125 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1126 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1127 !op_array->function_name) {
1128 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1129 }
1130 break;
1131 }
1132 }
1133 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1134 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1135 call_level++;
1136 stack_top += zend_jit_trace_frame_size(p->op_array);
1137 if (stack_top > stack_size) {
1138 stack_size = stack_top;
1139 }
1140 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1141 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1142 if (p->func != (zend_function*)&zend_pass_function
1143 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1144 || zend_string_equals_literal(p->func->common.function_name, "compact")
1145 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1146 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1147 }
1148 }
1149 frame_size = zend_jit_trace_frame_size(p->op_array);
1150 if (call_level == 0) {
1151 if (stack_top + frame_size > stack_size) {
1152 stack_size = stack_top + frame_size;
1153 }
1154 } else {
1155 call_level--;
1156 stack_top -= frame_size;
1157 }
1158 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1159 op_array = p->op_array;
1160 if (call_level == 0) {
1161 stack_top += zend_jit_trace_frame_size(op_array);
1162 if (stack_top > stack_size) {
1163 stack_size = stack_top;
1164 }
1165 } else {
1166 call_level--;
1167 }
1168 level++;
1169 jit_extension =
1170 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1171 ssa = &jit_extension->func_info.ssa;
1172 if (ssa->cfg.blocks_count) {
1173 /* pass */
1174 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1175 /* Too many functions in single trace */
1176 *num_op_arrays_ptr = num_op_arrays;
1177 return NULL;
1178 } else {
1179 /* Remember op_array to cleanup */
1180 op_arrays[num_op_arrays++] = op_array;
1181 /* Build SSA */
1182 ssa = zend_jit_trace_build_ssa(op_array, script);
1183 }
1184 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1185 if (level == 0) {
1186 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1187 jit_extension =
1188 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1189 ssa = &jit_extension->func_info.ssa;
1190 if (ssa->cfg.blocks_count) {
1191 /* pass */
1192 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1193 /* Too many functions in single trace */
1194 *num_op_arrays_ptr = num_op_arrays;
1195 return NULL;
1196 } else {
1197 /* Remember op_array to cleanup */
1198 op_arrays[num_op_arrays++] = op_array;
1199 /* Build SSA */
1200 ssa = zend_jit_trace_build_ssa(op_array, script);
1201 }
1202 } else {
1203 stack_top -= zend_jit_trace_frame_size(op_array);
1204 level--;
1205 }
1206 op_array = p->op_array;
1207 } else if (p->op == ZEND_JIT_TRACE_END) {
1208 break;
1209 }
1210 }
1211 *num_op_arrays_ptr = num_op_arrays;
1212
1213 /* Allocate space for abstract stack */
1214 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1215
1216 /* 2. Construct TSSA */
1217 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1218 tssa->cfg.flags = ZEND_SSA_TSSA;
1219 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1220 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1221 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1222
1223 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1224 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1225 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1226 tssa->cfg.blocks_count = 2;
1227 tssa->cfg.edges_count = 2;
1228
1229 tssa->cfg.predecessors[0] = 0;
1230 tssa->cfg.predecessors[1] = 1;
1231
1232 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1233 tssa->cfg.blocks[0].successors_count = 1;
1234 tssa->cfg.blocks[0].predecessors_count = 0;
1235 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1236 tssa->cfg.blocks[0].successors[0] = 1;
1237
1238 tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1239 tssa->cfg.blocks[1].successors_count = 1;
1240 tssa->cfg.blocks[1].predecessors_count = 2;
1241 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1242 tssa->cfg.blocks[1].successors[1] = 1;
1243 } else {
1244 tssa->cfg.blocks_count = 1;
1245 tssa->cfg.edges_count = 0;
1246
1247 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1248 tssa->cfg.blocks[0].successors_count = 0;
1249 tssa->cfg.blocks[0].predecessors_count = 0;
1250 }
1251 ((zend_tssa*)tssa)->used_stack = -1;
1252
1253 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1254 return tssa;
1255 }
1256
1257 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1258 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1259 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1260 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1261 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1262
1263 op_array = trace_buffer->op_array;
1264 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1265 ssa_vars_count = op_array->last_var;
1266 } else {
1267 ssa_vars_count = op_array->last_var + op_array->T;
1268 }
1269 stack = frame->stack;
1270 for (i = 0; i < ssa_vars_count; i++) {
1271 SET_STACK_VAR(stack, i, i);
1272 }
1273
1274 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1275 // TODO: For tracing, it's possible, to create pseudo Phi functions
1276 // at the end of loop, without this additional pass (like LuaJIT) ???
1277 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1278 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1279 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1280 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1281 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1282 }
1283
1284 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1285 idx = 0;
1286 level = 0;
1287 for (;;p++) {
1288 if (p->op == ZEND_JIT_TRACE_VM) {
1289 opline = p->opline;
1290 ssa_opcodes[idx] = opline;
1291 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1292 idx++;
1293 len = zend_jit_trace_op_len(p->opline);
1294 while (len > 1) {
1295 opline++;
1296 ssa_opcodes[idx] = opline;
1297 if (opline->opcode != ZEND_OP_DATA) {
1298 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1299 }
1300 idx++;
1301 len--;
1302 }
1303 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1304 frame = zend_jit_trace_call_frame(frame, op_array);
1305 stack = frame->stack;
1306 op_array = p->op_array;
1307 level++;
1308 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1309 return NULL;
1310 }
1311 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1312 for (i = 0; i < op_array->last_var; i++) {
1313 SET_STACK_VAR(stack, i, ssa_vars_count++);
1314 }
1315 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1316 op_array = p->op_array;
1317 frame = zend_jit_trace_ret_frame(frame, op_array);
1318 stack = frame->stack;
1319 if (level == 0) {
1320 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1321 return NULL;
1322 }
1323 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1324 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1325 SET_STACK_VAR(stack, i, ssa_vars_count++);
1326 }
1327 } else {
1328 level--;
1329 }
1330 } else if (p->op == ZEND_JIT_TRACE_END) {
1331 break;
1332 }
1333 }
1334
1335 op_array = trace_buffer->op_array;
1336 tssa->vars_count = ssa_vars_count;
1337 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1338 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1339 vars_count = op_array->last_var;
1340 } else {
1341 vars_count = op_array->last_var + op_array->T;
1342 }
1343 i = 0;
1344 while (i < vars_count) {
1345 ssa_vars[i].var = i;
1346 ssa_vars[i].scc = -1;
1347 ssa_vars[i].definition = -1;
1348 ssa_vars[i].use_chain = -1;
1349 i++;
1350 }
1351 while (i < tssa->vars_count) {
1352 ssa_vars[i].var = -1;
1353 ssa_vars[i].scc = -1;
1354 ssa_vars[i].definition = -1;
1355 ssa_vars[i].use_chain = -1;
1356 i++;
1357 }
1358
1359 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1360 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1361 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1362 /* Update Phi sources */
1363 zend_ssa_phi *phi = tssa->blocks[1].phis;
1364
1365 while (phi) {
1366 phi->sources[1] = STACK_VAR(stack, phi->var);
1367 ssa_vars[phi->ssa_var].var = phi->var;
1368 ssa_vars[phi->ssa_var].definition_phi = phi;
1369 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1370 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1371 phi = phi->next;
1372 }
1373 }
1374
1375 /* 3. Compute use-def chains */
1376 idx = (ssa_ops_count - 1);
1377 op = ssa_ops + idx;
1378 while (idx >= 0) {
1379 opline = ssa_opcodes[idx];
1380 if (op->op1_use >= 0) {
1381 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1382 ssa_vars[op->op1_use].use_chain = idx;
1383 }
1384 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1385 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1386 ssa_vars[op->op2_use].use_chain = idx;
1387 }
1388 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1389 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1390 ssa_vars[op->result_use].use_chain = idx;
1391 }
1392 if (op->op1_def >= 0) {
1393 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1394 ssa_vars[op->op1_def].definition = idx;
1395 }
1396 if (op->op2_def >= 0) {
1397 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1398 ssa_vars[op->op2_def].definition = idx;
1399 }
1400 if (op->result_def >= 0) {
1401 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1402 ssa_vars[op->result_def].definition = idx;
1403 }
1404 op--;
1405 idx--;
1406 }
1407
1408 /* 4. Type inference */
1409 op_array = trace_buffer->op_array;
1410 jit_extension =
1411 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1412 ssa = &jit_extension->func_info.ssa;
1413
1414 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1415
1416 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1417 i = 0;
1418 while (i < op_array->last_var) {
1419 if (i < op_array->num_args) {
1420 if (ssa->var_info
1421 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1422 /* pass */
1423 } else {
1424 if (ssa->vars) {
1425 ssa_vars[i].no_val = ssa->vars[i].no_val;
1426 ssa_vars[i].alias = ssa->vars[i].alias;
1427 } else {
1428 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1429 }
1430 if (op_array->arg_info) {
1431 zend_arg_info *arg_info = &op_array->arg_info[i];
1432 zend_class_entry *ce;
1433 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1434
1435 if (ZEND_ARG_SEND_MODE(arg_info)) {
1436 tmp |= MAY_BE_REF;
1437 }
1438 ssa_var_info[i].type = tmp;
1439 ssa_var_info[i].ce = ce;
1440 ssa_var_info[i].is_instanceof = 1;
1441 } else {
1442 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1443 }
1444 }
1445 } else {
1446 if (ssa->vars) {
1447 ssa_vars[i].no_val = ssa->vars[i].no_val;
1448 ssa_vars[i].alias = ssa->vars[i].alias;
1449 } else {
1450 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1451 }
1452 if (ssa_vars[i].alias == NO_ALIAS) {
1453 ssa_var_info[i].type = MAY_BE_UNDEF;
1454 } else {
1455 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1456 }
1457 }
1458 i++;
1459 }
1460 } else {
1461 int parent_vars_count = 0;
1462 zend_jit_trace_stack *parent_stack = NULL;
1463
1464 i = 0;
1465 if (parent_trace) {
1466 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1467 op_array->last_var + op_array->T);
1468 if (parent_vars_count) {
1469 parent_stack =
1470 zend_jit_traces[parent_trace].stack_map +
1471 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1472 }
1473 }
1474 while (i < op_array->last_var + op_array->T) {
1475 if (!ssa->var_info
1476 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1477 if (ssa->vars && i < ssa->vars_count) {
1478 ssa_vars[i].alias = ssa->vars[i].alias;
1479 } else {
1480 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1481 }
1482 if (i < op_array->last_var) {
1483 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1484 } else {
1485 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1486 }
1487 }
1488 if (i < parent_vars_count) {
1489 /* Initialize TSSA variable from parent trace */
1490 zend_uchar op_type = STACK_TYPE(parent_stack, i);
1491
1492 if (op_type != IS_UNKNOWN) {
1493 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1494 if (!ssa_var_info[i].type
1495 && op_type == IS_UNDEF
1496 && i >= op_array->last_var) {
1497 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1498 ssa_var_info[i].type |= MAY_BE_UNDEF;
1499 }
1500 }
1501 }
1502 i++;
1503 }
1504 }
1505
1506 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1507 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1508 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1509 /* Propagate initial value through Phi functions */
1510 zend_ssa_phi *phi = tssa->blocks[1].phis;
1511
1512 while (phi) {
1513 if (!ssa->var_info
1514 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) {
1515 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1516 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1517 }
1518 phi = phi->next;
1519 }
1520 }
1521
1522 frame = JIT_G(current_frame);
1523 top = zend_jit_trace_call_frame(frame, op_array);
1524 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1525 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1526 frame->used_stack = 0;
1527 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1528 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
1529 }
1530 memset(&return_value_info, 0, sizeof(return_value_info));
1531
1532 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1533 max_used_stack = used_stack = 0;
1534 } else {
1535 max_used_stack = used_stack = -1;
1536 }
1537
1538 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1539 idx = 0;
1540 level = 0;
1541 for (;;p++) {
1542 if (p->op == ZEND_JIT_TRACE_VM) {
1543 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1544 // zend_class_entry *op1_ce = NULL;
1545 zend_class_entry *op2_ce = NULL;
1546
1547 // TODO: range inference ???
1548 opline = p->opline;
1549
1550 op1_type = orig_op1_type = p->op1_type;
1551 op2_type = orig_op2_type = p->op2_type;
1552 op3_type = p->op3_type;
1553 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1554 op1_type = IS_UNKNOWN;
1555 }
1556 if (op1_type != IS_UNKNOWN) {
1557 op1_type &= ~IS_TRACE_PACKED;
1558 }
1559 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1560 op2_type = IS_UNKNOWN;
1561 }
1562 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1563 op3_type = IS_UNKNOWN;
1564 }
1565
1566 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1567 // op1_ce = (zend_class_entry*)(p+1)->ce;
1568 p++;
1569 }
1570 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1571 op2_ce = (zend_class_entry*)(p+1)->ce;
1572 p++;
1573 }
1574
1575 switch (opline->opcode) {
1576 case ZEND_ASSIGN_OP:
1577 if (opline->extended_value == ZEND_POW
1578 || opline->extended_value == ZEND_DIV) {
1579 // TODO: check for division by zero ???
1580 break;
1581 }
1582 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1583 break;
1584 }
1585 ADD_OP1_TRACE_GUARD();
1586 ADD_OP2_TRACE_GUARD();
1587 break;
1588 case ZEND_ASSIGN_DIM_OP:
1589 if (opline->extended_value == ZEND_POW
1590 || opline->extended_value == ZEND_DIV) {
1591 // TODO: check for division by zero ???
1592 break;
1593 }
1594 if (opline->result_type != IS_UNUSED) {
1595 break;
1596 }
1597 if (op3_type != IS_UNKNOWN
1598 && !zend_jit_supported_binary_op(
1599 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1600 break;
1601 }
1602 /* break missing intentionally */
1603 case ZEND_ASSIGN_DIM:
1604 if (opline->op1_type == IS_CV) {
1605 if ((opline+1)->op1_type == IS_CV
1606 && (opline+1)->op1.var == opline->op1.var) {
1607 /* skip $a[x] = $a; */
1608 break;
1609 }
1610 ADD_OP1_DATA_TRACE_GUARD();
1611 ADD_OP2_TRACE_GUARD();
1612 ADD_OP1_TRACE_GUARD();
1613 } else if (orig_op1_type != IS_UNKNOWN
1614 && (orig_op1_type & IS_TRACE_INDIRECT)
1615 && opline->result_type == IS_UNUSED) {
1616 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1617 ADD_OP1_DATA_TRACE_GUARD();
1618 }
1619 ADD_OP2_TRACE_GUARD();
1620 }
1621 break;
1622 case ZEND_ASSIGN_OBJ_OP:
1623 if (opline->extended_value == ZEND_POW
1624 || opline->extended_value == ZEND_DIV) {
1625 // TODO: check for division by zero ???
1626 break;
1627 }
1628 if (opline->result_type != IS_UNUSED) {
1629 break;
1630 }
1631 /* break missing intentionally */
1632 case ZEND_ASSIGN_OBJ:
1633 case ZEND_PRE_INC_OBJ:
1634 case ZEND_PRE_DEC_OBJ:
1635 case ZEND_POST_INC_OBJ:
1636 case ZEND_POST_DEC_OBJ:
1637 if (opline->op2_type != IS_CONST
1638 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1639 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1640 break;
1641 }
1642 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1643 if (opline->op1_type == IS_CV
1644 && (opline+1)->op1_type == IS_CV
1645 && (opline+1)->op1.var == opline->op1.var) {
1646 /* skip $a->prop += $a; */
1647 break;
1648 }
1649 ADD_OP1_DATA_TRACE_GUARD();
1650 }
1651 ADD_OP1_TRACE_GUARD();
1652 break;
1653 case ZEND_CONCAT:
1654 case ZEND_FAST_CONCAT:
1655 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1656 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1657 ADD_OP2_TRACE_GUARD();
1658 ADD_OP1_TRACE_GUARD();
1659 }
1660 break;
1661 case ZEND_IS_EQUAL:
1662 case ZEND_IS_NOT_EQUAL:
1663 case ZEND_IS_SMALLER:
1664 case ZEND_IS_SMALLER_OR_EQUAL:
1665 case ZEND_CASE:
1666 case ZEND_IS_IDENTICAL:
1667 case ZEND_IS_NOT_IDENTICAL:
1668 case ZEND_CASE_STRICT:
1669 case ZEND_BW_OR:
1670 case ZEND_BW_AND:
1671 case ZEND_BW_XOR:
1672 case ZEND_SL:
1673 case ZEND_SR:
1674 case ZEND_MOD:
1675 case ZEND_ADD:
1676 case ZEND_SUB:
1677 case ZEND_MUL:
1678 // case ZEND_DIV: // TODO: check for division by zero ???
1679 ADD_OP2_TRACE_GUARD();
1680 /* break missing intentionally */
1681 case ZEND_ECHO:
1682 case ZEND_STRLEN:
1683 case ZEND_QM_ASSIGN:
1684 case ZEND_FE_RESET_R:
1685 case ZEND_FE_FETCH_R:
1686 ADD_OP1_TRACE_GUARD();
1687 break;
1688 case ZEND_VERIFY_RETURN_TYPE:
1689 if (opline->op1_type == IS_UNUSED) {
1690 /* Always throws */
1691 break;
1692 }
1693 if (opline->op1_type == IS_CONST) {
1694 /* TODO Different instruction format, has return value */
1695 break;
1696 }
1697 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1698 /* Not worth bothering with */
1699 break;
1700 }
1701 ADD_OP1_TRACE_GUARD();
1702 break;
1703 case ZEND_FETCH_DIM_FUNC_ARG:
1704 if (!frame
1705 || !frame->call
1706 || !frame->call->func
1707 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1708 break;
1709 }
1710 ADD_OP2_TRACE_GUARD();
1711 ADD_OP1_TRACE_GUARD();
1712 break;
1713 case ZEND_PRE_INC:
1714 case ZEND_PRE_DEC:
1715 case ZEND_POST_INC:
1716 case ZEND_POST_DEC:
1717 if (opline->op1_type != IS_CV) {
1718 break;
1719 }
1720 ADD_OP1_TRACE_GUARD();
1721 break;
1722 case ZEND_ASSIGN:
1723 if (opline->op1_type != IS_CV) {
1724 break;
1725 }
1726 ADD_OP2_TRACE_GUARD();
1727 if (op1_type != IS_UNKNOWN
1728 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1729 ADD_OP1_TRACE_GUARD();
1730 }
1731 break;
1732 case ZEND_CAST:
1733 if (opline->extended_value != op1_type) {
1734 break;
1735 }
1736 ADD_OP1_TRACE_GUARD();
1737 break;
1738 case ZEND_JMPZ:
1739 case ZEND_JMPNZ:
1740 case ZEND_JMPZNZ:
1741 case ZEND_JMPZ_EX:
1742 case ZEND_JMPNZ_EX:
1743 case ZEND_BOOL:
1744 case ZEND_BOOL_NOT:
1745 ADD_OP1_TRACE_GUARD();
1746 break;
1747 case ZEND_ISSET_ISEMPTY_CV:
1748 if ((opline->extended_value & ZEND_ISEMPTY)) {
1749 // TODO: support for empty() ???
1750 break;
1751 }
1752 ADD_OP1_TRACE_GUARD();
1753 break;
1754 case ZEND_IN_ARRAY:
1755 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
1756 break;
1757 }
1758 ADD_OP1_TRACE_GUARD();
1759 break;
1760 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
1761 if ((opline->extended_value & ZEND_ISEMPTY)) {
1762 // TODO: support for empty() ???
1763 break;
1764 }
1765 /* break missing intentionally */
1766 case ZEND_FETCH_DIM_R:
1767 case ZEND_FETCH_DIM_IS:
1768 case ZEND_FETCH_LIST_R:
1769 ADD_OP1_TRACE_GUARD();
1770 ADD_OP2_TRACE_GUARD();
1771
1772 if (opline->op1_type != IS_CONST
1773 && op1_type == IS_ARRAY
1774 && ((opline->op2_type == IS_CONST
1775 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1776 || (opline->op2_type != IS_CONST
1777 && op2_type == IS_LONG))) {
1778
1779 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1780
1781 if ((info->type & MAY_BE_ARRAY_PACKED)
1782 && (info->type & MAY_BE_ARRAY_HASH)
1783 && orig_op1_type != IS_UNKNOWN
1784 && !(orig_op1_type & IS_TRACE_REFERENCE)) {
1785 info->type |= MAY_BE_PACKED_GUARD;
1786 if (orig_op1_type & IS_TRACE_PACKED) {
1787 info->type &= ~MAY_BE_ARRAY_HASH;
1788 } else {
1789 info->type &= ~MAY_BE_ARRAY_PACKED;
1790 }
1791 }
1792 }
1793 break;
1794 case ZEND_FETCH_DIM_W:
1795 case ZEND_FETCH_DIM_RW:
1796 // case ZEND_FETCH_DIM_UNSET:
1797 case ZEND_FETCH_LIST_W:
1798 if (opline->op1_type != IS_CV
1799 && (orig_op1_type == IS_UNKNOWN
1800 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
1801 break;
1802 }
1803 ADD_OP1_TRACE_GUARD();
1804 ADD_OP2_TRACE_GUARD();
1805 break;
1806 case ZEND_SEND_VAL_EX:
1807 case ZEND_SEND_VAR_EX:
1808 case ZEND_SEND_VAR_NO_REF_EX:
1809 if (opline->op2_type == IS_CONST) {
1810 /* Named parameters not supported in JIT */
1811 break;
1812 }
1813 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
1814 goto propagate_arg;
1815 }
1816 /* break missing intentionally */
1817 case ZEND_SEND_VAL:
1818 case ZEND_SEND_VAR:
1819 case ZEND_SEND_VAR_NO_REF:
1820 case ZEND_SEND_FUNC_ARG:
1821 if (opline->op2_type == IS_CONST) {
1822 /* Named parameters not supported in JIT */
1823 break;
1824 }
1825 ADD_OP1_TRACE_GUARD();
1826 propagate_arg:
1827 /* Propagate argument type */
1828 if (frame->call
1829 && frame->call->func
1830 && frame->call->func->type == ZEND_USER_FUNCTION
1831 && opline->op2.num <= frame->call->func->op_array.num_args) {
1832 uint32_t info;
1833
1834 if (opline->op1_type == IS_CONST) {
1835 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
1836 } else {
1837 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
1838 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
1839 }
1840 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
1841 zend_arg_info *arg_info;
1842
1843 ZEND_ASSERT(frame->call->func->op_array.arg_info);
1844 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
1845 if (ZEND_TYPE_IS_SET(arg_info->type)) {
1846 zend_class_entry *ce;
1847 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1848 info &= tmp;
1849 if (!info) {
1850 break;
1851 }
1852 }
1853 }
1854 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
1855 info |= MAY_BE_RCN;
1856 }
1857 if (info & MAY_BE_UNDEF) {
1858 info |= MAY_BE_NULL;
1859 info &= ~MAY_BE_UNDEF;
1860 }
1861 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
1862 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
1863 }
1864 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
1865 }
1866 break;
1867 case ZEND_RETURN:
1868 ADD_OP1_TRACE_GUARD();
1869 /* Propagate return value types */
1870 if (opline->op1_type == IS_UNUSED) {
1871 return_value_info.type = MAY_BE_NULL;
1872 } else if (opline->op1_type == IS_CONST) {
1873 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
1874 } else {
1875 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
1876 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
1877 if (return_value_info.type & MAY_BE_UNDEF) {
1878 return_value_info.type &= ~MAY_BE_UNDEF;
1879 return_value_info.type |= MAY_BE_NULL;
1880 }
1881 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
1882 /* CVs are going to be destructed and the reference-counter
1883 of return value may be decremented to 1 */
1884 return_value_info.type |= MAY_BE_RC1;
1885 }
1886 return_value_info.type &= ~MAY_BE_GUARD;
1887 }
1888 break;
1889 case ZEND_CHECK_FUNC_ARG:
1890 if (!frame
1891 || !frame->call
1892 || !frame->call->func) {
1893 break;
1894 }
1895 if (opline->op2_type == IS_CONST
1896 || opline->op2.num > MAX_ARG_FLAG_NUM) {
1897 /* Named parameters not supported in JIT */
1898 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
1899 break;
1900 }
1901 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
1902 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
1903 } else {
1904 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
1905 }
1906 break;
1907 case ZEND_FETCH_OBJ_FUNC_ARG:
1908 if (!frame
1909 || !frame->call
1910 || !frame->call->func
1911 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1912 break;
1913 }
1914 /* break missing intentionally */
1915 case ZEND_FETCH_OBJ_R:
1916 case ZEND_FETCH_OBJ_IS:
1917 case ZEND_FETCH_OBJ_W:
1918 if (opline->op2_type != IS_CONST
1919 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1920 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1921 break;
1922 }
1923 ADD_OP1_TRACE_GUARD();
1924 break;
1925 case ZEND_INIT_METHOD_CALL:
1926 if (opline->op2_type != IS_CONST
1927 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
1928 break;
1929 }
1930 ADD_OP1_TRACE_GUARD();
1931 break;
1932 case ZEND_INIT_DYNAMIC_CALL:
1933 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
1934 ADD_OP2_TRACE_GUARD();
1935 }
1936 break;
1937 case ZEND_SEND_ARRAY:
1938 case ZEND_SEND_UNPACK:
1939 case ZEND_CHECK_UNDEF_ARGS:
1940 case ZEND_INCLUDE_OR_EVAL:
1941 max_used_stack = used_stack = -1;
1942 case ZEND_TYPE_CHECK:
1943 if (opline->extended_value == MAY_BE_RESOURCE) {
1944 // TODO: support for is_resource() ???
1945 break;
1946 }
1947 if (op1_type != IS_UNKNOWN
1948 && (opline->extended_value == (1 << op1_type)
1949 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
1950 /* add guards only for exact checks, to avoid code duplication */
1951 ADD_OP1_TRACE_GUARD();
1952 }
1953 break;
1954 default:
1955 break;
1956 }
1957 len = zend_jit_trace_op_len(opline);
1958 if (ssa->var_info) {
1959 /* Add statically inferred ranges */
1960 if (ssa_ops[idx].op1_def >= 0) {
1961 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
1962 }
1963 if (ssa_ops[idx].op2_def >= 0) {
1964 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
1965 }
1966 if (ssa_ops[idx].result_def >= 0) {
1967 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
1968 }
1969 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
1970 if (ssa_ops[idx+1].op1_def >= 0) {
1971 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
1972 }
1973 if (ssa_ops[idx+1].op2_def >= 0) {
1974 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
1975 }
1976 if (ssa_ops[idx+1].result_def >= 0) {
1977 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
1978 }
1979 }
1980 } else {
1981 if (ssa_ops[idx].op1_def >= 0) {
1982 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
1983 }
1984 if (ssa_ops[idx].op2_def >= 0) {
1985 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
1986 }
1987 if (ssa_ops[idx].result_def >= 0) {
1988 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
1989 }
1990 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
1991 if (ssa_ops[idx+1].op1_def >= 0) {
1992 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
1993 }
1994 if (ssa_ops[idx+1].op2_def >= 0) {
1995 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
1996 }
1997 if (ssa_ops[idx+1].result_def >= 0) {
1998 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
1999 }
2000 }
2001 }
2002 if (opline->opcode == ZEND_RECV_INIT
2003 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2004 /* RECV_INIT always copy the constant */
2005 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2006 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2007 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2008 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2009 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2010 }
2011 } else {
2012 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2013 // TODO:
2014 assert(0);
2015 }
2016 }
2017 if (ssa->var_info) {
2018 /* Add statically inferred restrictions */
2019 if (ssa_ops[idx].op1_def >= 0) {
2020 if (opline->opcode == ZEND_SEND_VAR_EX
2021 && frame
2022 && frame->call
2023 && frame->call->func
2024 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2025 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2026 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2027 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2028 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2029 }
2030 } else {
2031 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2032 }
2033 }
2034 if (ssa_ops[idx].op2_def >= 0) {
2035 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2036 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2037 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2038 }
2039 }
2040 if (ssa_ops[idx].result_def >= 0) {
2041 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2042 }
2043 }
2044 idx++;
2045 while (len > 1) {
2046 opline++;
2047 if (opline->opcode != ZEND_OP_DATA) {
2048 if (ssa->var_info) {
2049 /* Add statically inferred ranges */
2050 if (ssa_ops[idx].op1_def >= 0) {
2051 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2052 }
2053 if (ssa_ops[idx].op2_def >= 0) {
2054 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2055 }
2056 if (ssa_ops[idx].result_def >= 0) {
2057 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2058 }
2059 } else {
2060 if (ssa_ops[idx].op1_def >= 0) {
2061 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2062 }
2063 if (ssa_ops[idx].op2_def >= 0) {
2064 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2065 }
2066 if (ssa_ops[idx].result_def >= 0) {
2067 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2068 }
2069 }
2070 if (opline->opcode == ZEND_RECV_INIT
2071 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2072 /* RECV_INIT always copy the constant */
2073 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2074 } else {
2075 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2076 // TODO:
2077 assert(0);
2078 }
2079 }
2080 }
2081 if (ssa->var_info) {
2082 /* Add statically inferred restrictions */
2083 if (ssa_ops[idx].op1_def >= 0) {
2084 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2085 }
2086 if (ssa_ops[idx].op2_def >= 0) {
2087 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2088 }
2089 if (ssa_ops[idx].result_def >= 0) {
2090 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2091 }
2092 }
2093 idx++;
2094 len--;
2095 }
2096
2097 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2098 op_array = p->op_array;
2099 jit_extension =
2100 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2101 ssa = &jit_extension->func_info.ssa;
2102
2103 call = frame->call;
2104 if (!call) {
2105 /* Trace missed INIT_FCALL opcode */
2106 call = top;
2107 TRACE_FRAME_INIT(call, op_array, 0, 0);
2108 call->used_stack = 0;
2109 top = zend_jit_trace_call_frame(top, op_array);
2110 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2111 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
2112 }
2113 } else {
2114 ZEND_ASSERT(&call->func->op_array == op_array);
2115 }
2116 frame->call = call->prev;
2117 call->prev = frame;
2118 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2119 frame = call;
2120
2121 level++;
2122 i = 0;
2123 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2124 while (i < op_array->last_var) {
2125 ssa_vars[v].var = i;
2126 if (i < op_array->num_args) {
2127 if (ssa->var_info
2128 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2129 /* pass */
2130 } else {
2131 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2132 if (op_array->arg_info) {
2133 zend_arg_info *arg_info = &op_array->arg_info[i];
2134 zend_class_entry *ce;
2135 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2136
2137 if (ZEND_ARG_SEND_MODE(arg_info)) {
2138 tmp |= MAY_BE_REF;
2139 }
2140 ssa_var_info[v].type = tmp;
2141 ssa_var_info[v].ce = ce;
2142 ssa_var_info[v].is_instanceof = 1;
2143 } else {
2144 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2145 }
2146 }
2147 } else {
2148 if (ssa->vars) {
2149 ssa_vars[v].no_val = ssa->vars[i].no_val;
2150 ssa_vars[v].alias = ssa->vars[i].alias;
2151 } else {
2152 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2153 }
2154 if (ssa_vars[v].alias == NO_ALIAS) {
2155 ssa_var_info[v].type = MAY_BE_UNDEF;
2156 } else {
2157 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2158 }
2159 }
2160 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2161 && i < op_array->num_args) {
2162 /* Propagate argument type */
2163 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2164 }
2165 i++;
2166 v++;
2167 }
2168 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2169 op_array = p->op_array;
2170 jit_extension =
2171 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2172 ssa = &jit_extension->func_info.ssa;
2173 if (level == 0) {
2174 i = 0;
2175 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2176 while (i < op_array->last_var) {
2177 ssa_vars[v].var = i;
2178 if (!ssa->var_info
2179 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2180 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2181 }
2182 i++;
2183 v++;
2184 }
2185 while (i < op_array->last_var + op_array->T) {
2186 ssa_vars[v].var = i;
2187 if (!ssa->var_info
2188 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2189 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2190 }
2191 i++;
2192 v++;
2193 }
2194 if (return_value_info.type != 0) {
2195 zend_jit_trace_rec *q = p + 1;
2196 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2197 q++;
2198 }
2199 if (q->op == ZEND_JIT_TRACE_VM
2200 || (q->op == ZEND_JIT_TRACE_END
2201 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2202 const zend_op *opline = q->opline - 1;
2203 if (opline->result_type != IS_UNUSED) {
2204 ssa_var_info[
2205 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2206 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2207 }
2208 }
2209 memset(&return_value_info, 0, sizeof(return_value_info));
2210 }
2211 } else {
2212 level--;
2213 if (return_value_info.type != 0) {
2214 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2215 const zend_op *opline = (p+1)->opline - 1;
2216 if (opline->result_type != IS_UNUSED) {
2217 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2218 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2219 }
2220 }
2221 }
2222 memset(&return_value_info, 0, sizeof(return_value_info));
2223 }
2224 }
2225
2226 top = frame;
2227 if (frame->prev) {
2228 if (used_stack > 0) {
2229 used_stack -= frame->used_stack;
2230 }
2231 frame = frame->prev;
2232 ZEND_ASSERT(&frame->func->op_array == op_array);
2233 } else {
2234 max_used_stack = used_stack = -1;
2235 frame = zend_jit_trace_ret_frame(frame, op_array);
2236 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2237 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2238 frame->used_stack = 0;
2239 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2240 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
2241 }
2242 }
2243
2244 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2245 call = top;
2246 TRACE_FRAME_INIT(call, p->func, 0, 0);
2247 call->prev = frame->call;
2248 call->used_stack = 0;
2249 frame->call = call;
2250 top = zend_jit_trace_call_frame(top, p->op_array);
2251 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2252 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2253 SET_STACK_INFO(call->stack, i, -1);
2254 }
2255 }
2256 if (used_stack >= 0
2257 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2258 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2259 max_used_stack = used_stack = -1;
2260 } else {
2261 const zend_op *opline = (p-1)->opline;
2262
2263 switch (opline->opcode) {
2264 case ZEND_INIT_FCALL:
2265 case ZEND_INIT_FCALL_BY_NAME:
2266 case ZEND_INIT_NS_FCALL_BY_NAME:
2267 case ZEND_INIT_METHOD_CALL:
2268 case ZEND_INIT_DYNAMIC_CALL:
2269 //case ZEND_INIT_STATIC_METHOD_CALL:
2270 //case ZEND_INIT_USER_CALL:
2271 //case ZEND_NEW:
2272 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2273 used_stack += frame->used_stack;
2274 if (used_stack > max_used_stack) {
2275 max_used_stack = used_stack;
2276 }
2277 break;
2278 default:
2279 max_used_stack = used_stack = -1;
2280 }
2281 }
2282 }
2283 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2284 call = frame->call;
2285 if (call) {
2286 top = call;
2287 frame->call = call->prev;
2288 }
2289 } else if (p->op == ZEND_JIT_TRACE_END) {
2290 break;
2291 }
2292 }
2293
2294 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2295
2296 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2297 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2298 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2299 /* Propagate guards through Phi sources */
2300 zend_ssa_phi *phi = tssa->blocks[1].phis;
2301
2302 op_array = trace_buffer->op_array;
2303 jit_extension =
2304 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2305 ssa = &jit_extension->func_info.ssa;
2306
2307 while (phi) {
2308 uint32_t t = ssa_var_info[phi->ssa_var].type;
2309 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2310 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2311
2312 if (t & MAY_BE_GUARD) {
2313 if (tssa->vars[phi->ssa_var].alias != NO_ALIAS) {
2314 /* pass */
2315 } else if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2316 if (!((t0 | t1) & MAY_BE_GUARD)) {
2317 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2318 }
2319 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2320 if (!(t1 & MAY_BE_GUARD)
2321 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2322 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2323 ssa_var_info[phi->sources[0]].type = t | MAY_BE_GUARD;
2324 }
2325 } else {
2326 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2327 ssa_var_info[phi->sources[0]].type = MAY_BE_GUARD | (t & t0);
2328 }
2329 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2330 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2331 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2332 ssa_var_info[phi->sources[1]].type = MAY_BE_GUARD | (t & t1);
2333 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2334 }
2335 }
2336 }
2337 }
2338 phi = phi->next;
2339 }
2340 }
2341
2342 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2343 if (parent_trace) {
2344 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2345 ZEND_JIT_TRACE_NUM,
2346 parent_trace,
2347 exit_num,
2348 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2349 trace_buffer->op_array->scope ? "::" : "",
2350 trace_buffer->op_array->function_name ?
2351 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2352 ZSTR_VAL(trace_buffer->op_array->filename),
2353 trace_buffer[1].opline->lineno);
2354 } else {
2355 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2356 ZEND_JIT_TRACE_NUM,
2357 zend_jit_trace_star_desc(trace_buffer->start),
2358 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2359 trace_buffer->op_array->scope ? "::" : "",
2360 trace_buffer->op_array->function_name ?
2361 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2362 ZSTR_VAL(trace_buffer->op_array->filename),
2363 trace_buffer[1].opline->lineno);
2364 }
2365 zend_jit_dump_trace(trace_buffer, tssa);
2366 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2367 uint32_t idx = trace_buffer[1].last;
2368 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2369 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2370 ZEND_JIT_TRACE_NUM,
2371 link_to);
2372 } else {
2373 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2374 ZEND_JIT_TRACE_NUM,
2375 zend_jit_trace_stop_description[trace_buffer->stop]);
2376 }
2377 }
2378
2379 return tssa;
2380 }
2381
zend_jit_close_var(zend_jit_trace_stack * stack,uint32_t n,int * start,int * end,uint8_t * flags,int line)2382 static void zend_jit_close_var(zend_jit_trace_stack *stack, uint32_t n, int *start, int *end, uint8_t *flags, int line)
2383 {
2384 int32_t var = STACK_VAR(stack, n);
2385
2386 if (var >= 0 && start[var] >= 0 && !(flags[var] & ZREG_LAST_USE)) {
2387 // TODO: shrink interval to last side exit ????
2388 end[var] = line;
2389 }
2390 }
2391
zend_jit_trace_use_var(int line,int var,int def,int use_chain,int * start,int * end,uint8_t * flags,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2392 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, int *start, int *end, uint8_t *flags, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2393 {
2394 ZEND_ASSERT(start[var] >= 0);
2395 ZEND_ASSERT(!(flags[var] & ZREG_LAST_USE));
2396 end[var] = line;
2397 if (def >= 0) {
2398 flags[var] |= ZREG_LAST_USE;
2399 } else if (use_chain < 0 && (flags[var] & (ZREG_LOAD|ZREG_STORE))) {
2400 flags[var] |= ZREG_LAST_USE;
2401 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2402 /* pass */
2403 } else if (op_array_ssa->vars) {
2404 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2405
2406 if (ssa->ops[line].op1_use == var) {
2407 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2408 flags[var] |= ZREG_LAST_USE;
2409 }
2410 } else if (ssa->ops[line].op2_use == var) {
2411 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2412 flags[var] |= ZREG_LAST_USE;
2413 }
2414 } else if (ssa->ops[line].result_use == var) {
2415 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2416 flags[var] |= ZREG_LAST_USE;
2417 }
2418 }
2419 }
2420 }
2421
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2422 static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2423 {
2424 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2425 zend_jit_trace_rec *p;
2426 const zend_op_array *op_array;
2427 zend_jit_op_array_trace_extension *jit_extension;
2428 const zend_ssa *op_array_ssa;
2429 const zend_ssa_op *ssa_op;
2430 int i, j, idx, count, level;
2431 int last_idx = -1;
2432 int *start, *end;
2433 uint8_t *flags;
2434 const zend_op_array **vars_op_array;
2435 zend_lifetime_interval **intervals, *list, *ival;
2436 void *checkpoint;
2437 zend_jit_trace_stack_frame *frame;
2438 zend_jit_trace_stack *stack;
2439 uint32_t parent_vars_count = parent_trace ?
2440 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2441 zend_jit_trace_stack *parent_stack = parent_trace ?
2442 zend_jit_traces[parent_trace].stack_map +
2443 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2444 ALLOCA_FLAG(use_heap);
2445
2446 ZEND_ASSERT(ssa->var_info != NULL);
2447
2448 start = do_alloca(sizeof(int) * ssa->vars_count * 2 +
2449 ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count) +
2450 ZEND_MM_ALIGNED_SIZE(sizeof(zend_op_array*) * ssa->vars_count),
2451 use_heap);
2452 if (!start) {
2453 return NULL;
2454 }
2455 end = start + ssa->vars_count;
2456 flags = (uint8_t*)(end + ssa->vars_count);
2457 vars_op_array = (const zend_op_array**)(flags + ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count));
2458
2459 memset(start, -1, sizeof(int) * ssa->vars_count * 2);
2460 memset(flags, 0, sizeof(uint8_t) * ssa->vars_count);
2461 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2462
2463 op_array = trace_buffer->op_array;
2464 jit_extension =
2465 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2466 op_array_ssa = &jit_extension->func_info.ssa;
2467 frame = JIT_G(current_frame);
2468 frame->prev = NULL;
2469 frame->func = (const zend_function*)op_array;
2470 stack = frame->stack;
2471
2472 count = 0;
2473
2474 i = 0;
2475 j = op_array->last_var;
2476 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2477 j += op_array->T;
2478 }
2479 while (i < j) {
2480 SET_STACK_VAR(stack, i, i);
2481 vars_op_array[i] = op_array;
2482 /* We don't start intervals for variables used in Phi */
2483 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2484 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2485 && ssa->vars[i].alias == NO_ALIAS
2486 && zend_jit_var_supports_reg(ssa, i)) {
2487 start[i] = 0;
2488 if (i < parent_vars_count
2489 && STACK_REG(parent_stack, i) != ZREG_NONE
2490 && STACK_REG(parent_stack, i) < ZREG_NUM) {
2491 /* We will try to reuse register from parent trace */
2492 flags[i] = STACK_FLAGS(parent_stack, i);
2493 count += 2;
2494 } else {
2495 flags[i] = ZREG_LOAD;
2496 count++;
2497 }
2498 }
2499 i++;
2500 }
2501
2502 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2503 j = op_array->last_var + op_array->T;
2504 while (i < j) {
2505 SET_STACK_VAR(stack, i, -1);
2506 i++;
2507 }
2508 }
2509
2510 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2511 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2512 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2513 zend_ssa_phi *phi = ssa->blocks[1].phis;
2514
2515 while (phi) {
2516 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2517 vars_op_array[phi->ssa_var] = op_array;
2518 if (ssa->vars[phi->ssa_var].use_chain >= 0
2519 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2520 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2521 start[phi->ssa_var] = 0;
2522 count++;
2523 }
2524 phi = phi->next;
2525 }
2526 }
2527
2528 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2529 level = 0;
2530 ssa_op = ssa->ops;
2531 idx = 0;
2532 for (;;p++) {
2533 if (p->op == ZEND_JIT_TRACE_VM) {
2534 const zend_op *opline = p->opline;
2535 int len;
2536 zend_bool support_opline;
2537
2538 support_opline =
2539 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2540
2541 if (support_opline
2542 && opline->opcode == ZEND_ASSIGN
2543 && opline->op1_type == IS_CV
2544 && ssa_op->op1_def >= 0
2545 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2546 /* avoid register allocation in case of possibility of indirect modification*/
2547 support_opline = 0;
2548 }
2549
2550 if (ssa_op->op1_use >= 0
2551 && start[ssa_op->op1_use] >= 0
2552 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2553 if (support_opline) {
2554 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2555 if (opline->op1_type != IS_CV) {
2556 if (opline->opcode == ZEND_CASE
2557 || opline->opcode == ZEND_CASE_STRICT
2558 || opline->opcode == ZEND_SWITCH_LONG
2559 || opline->opcode == ZEND_MATCH
2560 || opline->opcode == ZEND_FETCH_LIST_R
2561 || opline->opcode == ZEND_COPY_TMP
2562 || opline->opcode == ZEND_SWITCH_STRING
2563 || opline->opcode == ZEND_FE_FETCH_R
2564 || opline->opcode == ZEND_FE_FETCH_RW
2565 || opline->opcode == ZEND_FETCH_LIST_W
2566 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2567 || opline->opcode == ZEND_BIND_LEXICAL
2568 || opline->opcode == ZEND_ROPE_ADD) {
2569 /* The value is kept alive and may be used outside of the trace */
2570 flags[ssa_op->op1_use] |= ZREG_STORE;
2571 } else {
2572 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2573 }
2574 }
2575 } else {
2576 start[ssa_op->op1_use] = -1;
2577 end[ssa_op->op1_use] = -1;
2578 count--;
2579 }
2580 }
2581 if (ssa_op->op2_use >= 0
2582 && ssa_op->op2_use != ssa_op->op1_use
2583 && start[ssa_op->op2_use] >= 0
2584 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2585 if (support_opline) {
2586 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2587 if (opline->op2_type != IS_CV) {
2588 flags[ssa_op->op2_use] |= ZREG_LAST_USE;
2589 }
2590 } else {
2591 start[ssa_op->op2_use] = -1;
2592 end[ssa_op->op2_use] = -1;
2593 count--;
2594 }
2595 }
2596 if (ssa_op->result_use >= 0
2597 && ssa_op->result_use != ssa_op->op1_use
2598 && ssa_op->result_use != ssa_op->op2_use
2599 && start[ssa_op->result_use] >= 0
2600 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2601 if (support_opline) {
2602 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2603 } else {
2604 start[ssa_op->result_use] = -1;
2605 end[ssa_op->result_use] = -1;
2606 count--;
2607 }
2608 }
2609
2610 if (ssa_op->op1_def >= 0) {
2611 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2612 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2613 }
2614 if (ssa_op->op2_def >= 0) {
2615 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op2.var), start, end, flags, idx);
2616 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
2617 }
2618 if (ssa_op->result_def >= 0) {
2619 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2620 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2621 }
2622
2623 if (support_opline) {
2624 if (ssa_op->result_def >= 0
2625 && (ssa->vars[ssa_op->result_def].use_chain >= 0
2626 || ssa->vars[ssa_op->result_def].phi_use_chain)
2627 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
2628 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
2629 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
2630 || opline->opcode == ZEND_PRE_INC
2631 || opline->opcode == ZEND_PRE_DEC
2632 || opline->opcode == ZEND_POST_INC
2633 || opline->opcode == ZEND_POST_DEC
2634 || opline->opcode == ZEND_ADD
2635 || opline->opcode == ZEND_SUB
2636 || opline->opcode == ZEND_MUL
2637 || opline->opcode == ZEND_FETCH_DIM_R) {
2638 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
2639 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
2640 start[ssa_op->result_def] = idx;
2641 vars_op_array[ssa_op->result_def] = op_array;
2642 count++;
2643 }
2644 }
2645 }
2646 if (ssa_op->op1_def >= 0
2647 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2648 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2649 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2650 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2651 start[ssa_op->op1_def] = idx;
2652 vars_op_array[ssa_op->op1_def] = op_array;
2653 count++;
2654 }
2655 if (ssa_op->op2_def >= 0
2656 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
2657 || ssa->vars[ssa_op->op2_def].phi_use_chain)
2658 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
2659 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)) {
2660 start[ssa_op->op2_def] = idx;
2661 vars_op_array[ssa_op->op2_def] = op_array;
2662 count++;
2663 }
2664 }
2665
2666 len = zend_jit_trace_op_len(opline);
2667 switch (opline->opcode) {
2668 case ZEND_ASSIGN_DIM:
2669 case ZEND_ASSIGN_OBJ:
2670 case ZEND_ASSIGN_STATIC_PROP:
2671 case ZEND_ASSIGN_DIM_OP:
2672 case ZEND_ASSIGN_OBJ_OP:
2673 case ZEND_ASSIGN_STATIC_PROP_OP:
2674 case ZEND_ASSIGN_OBJ_REF:
2675 case ZEND_ASSIGN_STATIC_PROP_REF:
2676 /* OP_DATA */
2677 ssa_op++;
2678 opline++;
2679 if (ssa_op->op1_use >= 0
2680 && start[ssa_op->op1_use] >= 0
2681 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2682 if (support_opline) {
2683 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2684 if (opline->op1_type != IS_CV) {
2685 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2686 }
2687 } else {
2688 start[ssa_op->op1_use] = -1;
2689 end[ssa_op->op1_use] = -1;
2690 count--;
2691 }
2692 }
2693 if (ssa_op->op1_def >= 0) {
2694 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2695 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2696 if (support_opline
2697 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2698 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2699 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2700 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2701 start[ssa_op->op1_def] = idx;
2702 vars_op_array[ssa_op->op1_def] = op_array;
2703 count++;
2704 }
2705 }
2706 ssa_op++;
2707 opline++;
2708 idx+=2;
2709 break;
2710 case ZEND_RECV_INIT:
2711 ssa_op++;
2712 opline++;
2713 idx++;
2714 while (opline->opcode == ZEND_RECV_INIT) {
2715 /* RECV_INIT doesn't support registers */
2716 if (ssa_op->result_def >= 0) {
2717 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2718 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2719 }
2720 ssa_op++;
2721 opline++;
2722 idx++;
2723 }
2724 break;
2725 case ZEND_BIND_GLOBAL:
2726 ssa_op++;
2727 opline++;
2728 idx++;
2729 while (opline->opcode == ZEND_BIND_GLOBAL) {
2730 /* BIND_GLOBAL doesn't support registers */
2731 if (ssa_op->op1_def >= 0) {
2732 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2733 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2734 }
2735 ssa_op++;
2736 opline++;
2737 idx++;
2738 }
2739 break;
2740 default:
2741 ssa_op += len;
2742 idx += len;
2743 break;
2744 }
2745 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2746 /* New call frames */
2747 zend_jit_trace_stack_frame *prev_frame = frame;
2748
2749 frame = zend_jit_trace_call_frame(frame, op_array);
2750 frame->prev = prev_frame;
2751 frame->func = (const zend_function*)p->op_array;
2752 stack = frame->stack;
2753 op_array = p->op_array;
2754 jit_extension =
2755 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2756 op_array_ssa = &jit_extension->func_info.ssa;
2757 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2758 for (i = 0; i < op_array->last_var; i++) {
2759 SET_STACK_VAR(stack, i, j);
2760 vars_op_array[j] = op_array;
2761 if (ssa->vars[j].use_chain >= 0
2762 && ssa->vars[j].alias == NO_ALIAS
2763 && zend_jit_var_supports_reg(ssa, j)) {
2764 start[j] = idx;
2765 flags[j] = ZREG_LOAD;
2766 count++;
2767 }
2768 j++;
2769 }
2770 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
2771 SET_STACK_VAR(stack, i, -1);
2772 }
2773 level++;
2774 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2775 /* Close exiting call frames */
2776 for (i = 0; i < op_array->last_var; i++) {
2777 zend_jit_close_var(stack, i, start, end, flags, idx-1);
2778 }
2779 op_array = p->op_array;
2780 jit_extension =
2781 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2782 op_array_ssa = &jit_extension->func_info.ssa;
2783 frame = zend_jit_trace_ret_frame(frame, op_array);
2784 stack = frame->stack;
2785 if (level == 0) {
2786 /* New return frames */
2787 frame->prev = NULL;
2788 frame->func = (const zend_function*)op_array;
2789 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2790 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2791 SET_STACK_VAR(stack, i, j);
2792 vars_op_array[j] = op_array;
2793 if (ssa->vars[j].use_chain >= 0
2794 && ssa->vars[j].alias == NO_ALIAS
2795 && zend_jit_var_supports_reg(ssa, j)) {
2796 start[j] = idx;
2797 flags[j] = ZREG_LOAD;
2798 count++;
2799 }
2800 j++;
2801 }
2802 } else {
2803 level--;
2804 }
2805 } else if (p->op == ZEND_JIT_TRACE_END) {
2806 break;
2807 }
2808 }
2809
2810 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2811 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2812 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2813 zend_ssa_phi *phi = ssa->blocks[1].phis;
2814
2815 while (phi) {
2816 i = phi->sources[1];
2817 if (start[i] >= 0 && !ssa->vars[phi->ssa_var].no_val) {
2818 end[i] = idx;
2819 flags[i] &= ~ZREG_LAST_USE;
2820 }
2821 phi = phi->next;
2822 }
2823
2824 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
2825 for (i = 0; i < op_array->last_var; i++) {
2826 if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) {
2827 end[i] = idx;
2828 flags[i] &= ~ZREG_LAST_USE;
2829 } else {
2830 zend_jit_close_var(stack, i, start, end, flags, idx);
2831 }
2832 }
2833 }
2834 } else {
2835 last_idx = idx;
2836 for (i = 0; i < op_array->last_var; i++) {
2837 zend_jit_close_var(stack, i, start, end, flags, idx);
2838 }
2839 while (frame->prev) {
2840 frame = frame->prev;
2841 op_array = &frame->func->op_array;
2842 stack = frame->stack;
2843 for (i = 0; i < op_array->last_var; i++) {
2844 zend_jit_close_var(stack, i, start, end, flags, idx);
2845 }
2846 }
2847 }
2848
2849 if (!count) {
2850 free_alloca(start, use_heap);
2851 return NULL;
2852 }
2853
2854 checkpoint = zend_arena_checkpoint(CG(arena));
2855 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval));
2856 memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count);
2857 list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count);
2858 j = 0;
2859 for (i = 0; i < ssa->vars_count; i++) {
2860 if (start[i] >= 0 && end[i] >= 0) {
2861 ZEND_ASSERT(j < count);
2862 if ((flags[i] & ZREG_LOAD) &&
2863 (flags[i] & ZREG_LAST_USE) &&
2864 end[i] == ssa->vars[i].use_chain) {
2865 /* skip life range with single use */
2866 continue;
2867 }
2868 intervals[i] = &list[j];
2869 list[j].ssa_var = i;
2870 list[j].reg = ZREG_NONE;
2871 list[j].flags = flags[i];
2872 list[j].range.start = start[i];
2873 list[j].range.end = end[i];
2874 list[j].range.next = NULL;
2875 list[j].hint = NULL;
2876 list[j].used_as_hint = NULL;
2877 list[j].list_next = NULL;
2878 j++;
2879 }
2880 }
2881 count = j;
2882 free_alloca(start, use_heap);
2883 start = end = NULL;
2884
2885 if (!count) {
2886 zend_arena_release(&CG(arena), checkpoint);
2887 return NULL;
2888 }
2889
2890 /* Add hints */
2891 if (parent_vars_count) {
2892 i = trace_buffer->op_array->last_var;
2893 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2894 i += trace_buffer->op_array->T;
2895 }
2896 if ((uint32_t)i > parent_vars_count) {
2897 i = parent_vars_count;
2898 }
2899 while (i > 0) {
2900 i--;
2901 if (intervals[i]
2902 && STACK_REG(parent_stack, i) != ZREG_NONE
2903 && STACK_REG(parent_stack, i) < ZREG_NUM) {
2904 list[j].ssa_var = - 1;
2905 list[j].reg = STACK_REG(parent_stack, i);
2906 list[j].flags = 0;
2907 list[j].range.start = -1;
2908 list[j].range.end = -1;
2909 list[j].range.next = NULL;
2910 list[j].hint = NULL;
2911 list[j].used_as_hint = NULL;
2912 list[j].list_next = NULL;
2913 intervals[i]->hint = &list[j];
2914 j++;
2915 }
2916 }
2917 }
2918
2919 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2920 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2921 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2922 zend_ssa_phi *phi = ssa->blocks[1].phis;
2923
2924 while (phi) {
2925 if (intervals[phi->ssa_var]) {
2926 if (intervals[phi->sources[1]]
2927 && (ssa->var_info[phi->sources[1]].type & MAY_BE_ANY) ==
2928 (ssa->var_info[phi->ssa_var].type & MAY_BE_ANY)) {
2929 intervals[phi->sources[1]]->hint = intervals[phi->ssa_var];
2930 }
2931 }
2932 phi = phi->next;
2933 }
2934 }
2935
2936 for (i = 0; i < ssa->vars_count; i++) {
2937 if (intervals[i] && !intervals[i]->hint) {
2938
2939 if (ssa->vars[i].definition >= 0) {
2940 uint32_t line = ssa->vars[i].definition;
2941 const zend_op *opline = ssa_opcodes[line];
2942
2943 switch (opline->opcode) {
2944 case ZEND_QM_ASSIGN:
2945 case ZEND_POST_INC:
2946 case ZEND_POST_DEC:
2947 if (ssa->ops[line].op1_use >= 0 &&
2948 intervals[ssa->ops[line].op1_use] &&
2949 (i == ssa->ops[line].op1_def ||
2950 (i == ssa->ops[line].result_def &&
2951 (ssa->ops[line].op1_def < 0 ||
2952 !intervals[ssa->ops[line].op1_def])))) {
2953 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
2954 }
2955 break;
2956 case ZEND_SEND_VAR:
2957 if (opline->op2_type == IS_CONST) {
2958 /* Named parameters not supported in JIT */
2959 break;
2960 }
2961 case ZEND_PRE_INC:
2962 case ZEND_PRE_DEC:
2963 if (i == ssa->ops[line].op1_def &&
2964 ssa->ops[line].op1_use >= 0 &&
2965 intervals[ssa->ops[line].op1_use]) {
2966 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
2967 }
2968 break;
2969 case ZEND_ASSIGN:
2970 if (ssa->ops[line].op2_use >= 0 &&
2971 intervals[ssa->ops[line].op2_use] &&
2972 (i == ssa->ops[line].op2_def ||
2973 (i == ssa->ops[line].op1_def &&
2974 (ssa->ops[line].op2_def < 0 ||
2975 !intervals[ssa->ops[line].op2_def])) ||
2976 (i == ssa->ops[line].result_def &&
2977 (ssa->ops[line].op2_def < 0 ||
2978 !intervals[ssa->ops[line].op2_def]) &&
2979 (ssa->ops[line].op1_def < 0 ||
2980 !intervals[ssa->ops[line].op1_def])))) {
2981 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
2982 }
2983 break;
2984 case ZEND_SUB:
2985 case ZEND_ADD:
2986 case ZEND_MUL:
2987 case ZEND_BW_OR:
2988 case ZEND_BW_AND:
2989 case ZEND_BW_XOR:
2990 if (i == ssa->ops[line].result_def) {
2991 if (ssa->ops[line].op1_use >= 0 &&
2992 intervals[ssa->ops[line].op1_use] &&
2993 ssa->ops[line].op1_use_chain < 0 &&
2994 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
2995 (ssa->var_info[i].type & MAY_BE_ANY) ==
2996 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
2997
2998 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op1_use].definition_phi;
2999 if (phi &&
3000 intervals[phi->sources[1]] &&
3001 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op1_use]) {
3002 break;
3003 }
3004 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3005 } else if (opline->opcode != ZEND_SUB &&
3006 ssa->ops[line].op2_use >= 0 &&
3007 intervals[ssa->ops[line].op2_use] &&
3008 ssa->ops[line].op2_use_chain < 0 &&
3009 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
3010 (ssa->var_info[i].type & MAY_BE_ANY) ==
3011 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
3012
3013 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op2_use].definition_phi;
3014 if (phi &&
3015 intervals[phi->sources[1]] &&
3016 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op2_use]) {
3017 break;
3018 }
3019 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3020 }
3021 }
3022 break;
3023 }
3024 }
3025 }
3026 }
3027
3028 list = zend_jit_sort_intervals(intervals, ssa->vars_count);
3029
3030 if (list) {
3031 ival = list;
3032 while (ival) {
3033 if (ival->hint) {
3034 ival->hint->used_as_hint = ival;
3035 }
3036 ival = ival->list_next;
3037 }
3038 }
3039
3040 if (list) {
3041 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3042 fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM);
3043 ival = list;
3044 while (ival) {
3045 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3046 ival = ival->list_next;
3047 }
3048 }
3049 }
3050
3051 /* Linear Scan Register Allocation (op_array is not actually used, only fn_flags matters) */
3052 list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list);
3053
3054 if (list) {
3055 zend_lifetime_interval *ival, *next;
3056
3057 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
3058 ival = list;
3059 count = 0;
3060 while (ival != NULL) {
3061 ZEND_ASSERT(ival->reg != ZREG_NONE);
3062 count++;
3063 next = ival->list_next;
3064 ival->list_next = intervals[ival->ssa_var];
3065 intervals[ival->ssa_var] = ival;
3066 ival = next;
3067 }
3068
3069 if (!count) {
3070 zend_arena_release(&CG(arena), checkpoint);
3071 return NULL;
3072 }
3073
3074 /* Add LOAD flag to registers that can't reuse register from parent trace */
3075 if (parent_vars_count) {
3076 i = trace_buffer->op_array->last_var;
3077 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3078 i += trace_buffer->op_array->T;
3079 }
3080 if ((uint32_t)i > parent_vars_count) {
3081 i = parent_vars_count;
3082 }
3083 while (i > 0) {
3084 i--;
3085 if (intervals[i] && intervals[i]->reg != STACK_REG(parent_stack, i)) {
3086 intervals[i]->flags |= ZREG_LOAD;
3087 }
3088 }
3089 }
3090
3091 /* SSA resolution */
3092 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3093 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3094 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3095 zend_ssa_phi *phi = ssa->blocks[1].phis;
3096
3097 while (phi) {
3098 int def = phi->ssa_var;
3099 int use = phi->sources[1];
3100
3101 if (intervals[def]) {
3102 if (!intervals[use]) {
3103 intervals[def]->flags |= ZREG_LOAD;
3104 if ((intervals[def]->flags & ZREG_LAST_USE)
3105 && ssa->vars[def].use_chain >= 0
3106 && ssa->vars[def].use_chain == intervals[def]->range.end) {
3107 /* remove interval used once */
3108 intervals[def] = NULL;
3109 count--;
3110 }
3111 } else if (intervals[def]->reg != intervals[use]->reg) {
3112 intervals[def]->flags |= ZREG_LOAD;
3113 if (ssa->vars[use].use_chain >= 0) {
3114 intervals[use]->flags |= ZREG_STORE;
3115 } else {
3116 intervals[use] = NULL;
3117 count--;
3118 }
3119 } else {
3120 use = phi->sources[0];
3121 ZEND_ASSERT(!intervals[use]);
3122 intervals[use] = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
3123 intervals[use]->ssa_var = phi->sources[0];
3124 intervals[use]->reg = intervals[def]->reg;
3125 intervals[use]->flags = ZREG_LOAD;
3126 intervals[use]->range.start = 0;
3127 intervals[use]->range.end = 0;
3128 intervals[use]->range.next = NULL;
3129 intervals[use]->hint = NULL;
3130 intervals[use]->used_as_hint = NULL;
3131 intervals[use]->list_next = NULL;
3132 }
3133 } else if (intervals[use]
3134 && (!ssa->vars[def].no_val
3135 || ssa->var_info[def].type != ssa->var_info[use].type)) {
3136 if (ssa->vars[use].use_chain >= 0) {
3137 intervals[use]->flags |= ZREG_STORE;
3138 } else {
3139 intervals[use] = NULL;
3140 count--;
3141 }
3142 }
3143 phi = phi->next;
3144 }
3145 } else {
3146 for (i = 0; i < ssa->vars_count; i++) {
3147 if (intervals[i]
3148 && intervals[i]->range.end == last_idx
3149 && !(intervals[i]->flags & (ZREG_LOAD|ZREG_STORE))) {
3150 intervals[i]->flags |= ZREG_STORE;
3151 }
3152 }
3153 }
3154
3155 if (!count) {
3156 zend_arena_release(&CG(arena), checkpoint);
3157 return NULL;
3158 }
3159
3160 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3161 fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM);
3162 for (i = 0; i < ssa->vars_count; i++) {
3163 ival = intervals[i];
3164 while (ival) {
3165 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3166 ival = ival->list_next;
3167 }
3168 }
3169 }
3170
3171 return intervals;
3172 }
3173
3174 zend_arena_release(&CG(arena), checkpoint); //???
3175 return NULL;
3176 }
3177
zend_jit_trace_clenup_stack(zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,zend_lifetime_interval ** ra)3178 static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, zend_lifetime_interval **ra)
3179 {
3180 uint32_t line = ssa_op - ssa->ops;
3181
3182 if (ssa_op->op1_use >= 0
3183 && ra[ssa_op->op1_use]
3184 && ra[ssa_op->op1_use]->range.end == line) {
3185 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
3186 }
3187 if (ssa_op->op2_use >= 0
3188 && ra[ssa_op->op2_use]
3189 && ra[ssa_op->op2_use]->range.end == line) {
3190 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op2.var), ZREG_NONE);
3191 }
3192 if (ssa_op->result_use >= 0
3193 && ra[ssa_op->result_use]
3194 && ra[ssa_op->result_use]->range.end == line) {
3195 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE);
3196 }
3197 }
3198
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3199 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3200 {
3201 zend_op *next_opline = (zend_op*)(opline + 1);
3202
3203 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3204 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3205 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3206 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3207 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3208 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3209 }
3210 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3211 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3212 }
3213 }
3214
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3215 static zend_bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3216 {
3217 int var = ssa_op->result_def;
3218 int i;
3219 int use = ssa->vars[var].use_chain;
3220 const zend_op *opline;
3221
3222 if (use < 0
3223 || ssa->vars[var].phi_use_chain
3224 || ssa->ops[use].op1_use != var
3225 || ssa->ops[use].op1_use_chain != -1) {
3226 return 0;
3227 }
3228
3229 opline = ssa_opcodes[use];
3230 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3231 return (opline->op2_type == IS_CONST &&
3232 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3233 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3234 if (!JIT_G(current_frame)
3235 || !JIT_G(current_frame)->call
3236 || !JIT_G(current_frame)->call->func
3237 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3238 return 0;
3239 }
3240 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3241 && opline->opcode != ZEND_FETCH_OBJ_IS
3242 && opline->opcode != ZEND_FETCH_OBJ_W
3243 && opline->opcode != ZEND_ASSIGN_OBJ
3244 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3245 && opline->opcode != ZEND_PRE_INC_OBJ
3246 && opline->opcode != ZEND_PRE_DEC_OBJ
3247 && opline->opcode != ZEND_POST_INC_OBJ
3248 && opline->opcode != ZEND_POST_DEC_OBJ) {
3249 return 0;
3250 }
3251
3252 if (opline->op2_type != IS_CONST
3253 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3254 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3255 return 0;
3256 }
3257
3258 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3259 if (opline->op1_type == IS_CV
3260 && (opline+1)->op1_type == IS_CV
3261 && (opline+1)->op1.var == opline->op1.var) {
3262 /* skip $a->prop += $a; */
3263 return 0;
3264 }
3265 if (!zend_jit_supported_binary_op(
3266 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3267 return 0;
3268 }
3269 }
3270
3271 for (i = ssa->vars[var].definition; i < use; i++) {
3272 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3273 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3274 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3275 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3276 return 0;
3277 }
3278 }
3279
3280 return 1;
3281 }
3282
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3283 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3284 {
3285 uint32_t i;
3286
3287 for (i = 0; i < stack_size; i++) {
3288 if (STACK_REG(stack, i) != ZREG_NONE
3289 && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
3290 return 1;
3291 }
3292 }
3293 return 0;
3294 }
3295
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3296 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3297 {
3298 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3299 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3300 uint32_t stack_size;
3301 zend_jit_trace_stack *stack;
3302
3303 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3304 return 1;
3305 }
3306
3307 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3308 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3309 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3310 }
3311
zend_jit_trace_deoptimization(dasm_State ** Dst,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_lifetime_interval ** ra,zend_bool polymorphic_side_trace)3312 static int zend_jit_trace_deoptimization(dasm_State **Dst,
3313 uint32_t flags,
3314 const zend_op *opline,
3315 zend_jit_trace_stack *parent_stack,
3316 int parent_vars_count,
3317 zend_ssa *ssa,
3318 zend_jit_trace_stack *stack,
3319 zend_lifetime_interval **ra,
3320 zend_bool polymorphic_side_trace)
3321 {
3322 int i;
3323 zend_bool has_constants = 0;
3324 zend_bool has_unsaved_vars = 0;
3325
3326 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3327 for (i = 0; i < parent_vars_count; i++) {
3328 int8_t reg = STACK_REG(parent_stack, i);
3329
3330 if (reg != ZREG_NONE) {
3331 if (reg < ZREG_NUM) {
3332 if (ssa && ssa->vars[i].no_val) {
3333 /* pass */
3334 } else if (ra && ra[i] && ra[i]->reg == reg) {
3335 /* register already loaded by parent trace */
3336 if (stack) {
3337 SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i));
3338 }
3339 has_unsaved_vars = 1;
3340 } else {
3341 uint8_t type = STACK_TYPE(parent_stack, i);
3342
3343 if (stack) {
3344 SET_STACK_TYPE(stack, i, type, 1);
3345 }
3346 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3347 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3348 STACK_MEM_TYPE(parent_stack, i) != type)) {
3349 return 0;
3350 }
3351 }
3352 } else {
3353 /* delay custom deoptimization instructions to prevent register clobbering */
3354 has_constants = 1;
3355 }
3356 }
3357 }
3358
3359 if (has_unsaved_vars
3360 && (has_constants
3361 || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) {
3362 for (i = 0; i < parent_vars_count; i++) {
3363 int8_t reg = STACK_REG(parent_stack, i);
3364
3365 if (reg != ZREG_NONE) {
3366 if (reg < ZREG_NUM) {
3367 if (ssa && ssa->vars[i].no_val) {
3368 /* pass */
3369 } else if (ra && ra[i] && ra[i]->reg == reg) {
3370 uint8_t type = STACK_TYPE(parent_stack, i);
3371
3372 if (stack) {
3373 SET_STACK_TYPE(stack, i, type, 1);
3374 }
3375 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3376 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3377 STACK_MEM_TYPE(parent_stack, i) != type)) {
3378 return 0;
3379 }
3380 }
3381 }
3382 }
3383 }
3384 }
3385
3386 if (has_constants) {
3387 for (i = 0; i < parent_vars_count; i++) {
3388 int8_t reg = STACK_REG(parent_stack, i);
3389
3390 if (reg != ZREG_NONE) {
3391 if (reg < ZREG_NUM) {
3392 /* pass */
3393 } else if (reg == ZREG_THIS) {
3394 if (polymorphic_side_trace) {
3395 ssa->var_info[i].delayed_fetch_this = 1;
3396 if (stack) {
3397 SET_STACK_REG(stack, i, ZREG_THIS);
3398 }
3399 } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
3400 return 0;
3401 }
3402 } else {
3403 if (reg == ZREG_ZVAL_COPY_R0
3404 &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) {
3405 return 0;
3406 }
3407 if (!zend_jit_store_const(Dst, i, reg)) {
3408 return 0;
3409 }
3410 }
3411 }
3412 }
3413 }
3414
3415 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3416 if (!zend_jit_save_call_chain(Dst, -1)) {
3417 return 0;
3418 }
3419 }
3420
3421 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3422 const zend_op *op = opline - 1;
3423
3424 if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) {
3425 return 0;
3426 }
3427 }
3428
3429 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3430 const zend_op *op = opline - 1;
3431
3432 if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) {
3433 return 0;
3434 }
3435 }
3436
3437 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3438 if (!zend_jit_check_exception(Dst)) {
3439 return 0;
3440 }
3441 }
3442
3443 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3444 if (!zend_jit_free_trampoline(Dst)) {
3445 return 0;
3446 }
3447 }
3448
3449 return 1;
3450 }
3451
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3452 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3453 {
3454 info->has_range = 1;
3455 info->range.min = min;
3456 info->range.max = max;
3457 info->range.underflow = 0;
3458 info->range.overflow = 0;
3459 }
3460
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,zend_bool exit_if_true)3461 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, zend_bool exit_if_true)
3462 {
3463 zend_long op1_min, op1_max, op2_min, op2_max;
3464
3465 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3466 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3467 return;
3468 }
3469
3470 op1_min = OP1_MIN_RANGE();
3471 op1_max = OP1_MAX_RANGE();
3472 op2_min = OP2_MIN_RANGE();
3473 op2_max = OP2_MAX_RANGE();
3474
3475 switch (opline->opcode) {
3476 case ZEND_IS_EQUAL:
3477 case ZEND_CASE:
3478 case ZEND_IS_IDENTICAL:
3479 case ZEND_CASE_STRICT:
3480 case ZEND_IS_NOT_IDENTICAL:
3481 if (!exit_if_true) {
3482 /* op1 == op2 */
3483 if (ssa_op->op1_use >= 0) {
3484 zend_jit_trace_set_var_range(
3485 &ssa->var_info[ssa_op->op1_use],
3486 MAX(op1_min, op2_min),
3487 MIN(op1_max, op2_max));
3488 }
3489 if (ssa_op->op2_use >= 0) {
3490 zend_jit_trace_set_var_range(
3491 &ssa->var_info[ssa_op->op2_use],
3492 MAX(op2_min, op1_min),
3493 MIN(op2_max, op1_max));
3494 }
3495 }
3496 break;
3497 case ZEND_IS_NOT_EQUAL:
3498 if (exit_if_true) {
3499 /* op1 == op2 */
3500 if (ssa_op->op1_use >= 0) {
3501 zend_jit_trace_set_var_range(
3502 &ssa->var_info[ssa_op->op1_use],
3503 MAX(op1_min, op2_min),
3504 MIN(op1_max, op2_max));
3505 }
3506 if (ssa_op->op2_use >= 0) {
3507 zend_jit_trace_set_var_range(
3508 &ssa->var_info[ssa_op->op2_use],
3509 MAX(op2_min, op1_min),
3510 MIN(op2_max, op1_max));
3511 }
3512 }
3513 break;
3514 case ZEND_IS_SMALLER_OR_EQUAL:
3515 if (!exit_if_true) {
3516 /* op1 <= op2 */
3517 if (ssa_op->op1_use >= 0) {
3518 zend_jit_trace_set_var_range(
3519 &ssa->var_info[ssa_op->op1_use],
3520 op1_min,
3521 MIN(op1_max, op2_max));
3522 }
3523 if (ssa_op->op2_use >= 0) {
3524 zend_jit_trace_set_var_range(
3525 &ssa->var_info[ssa_op->op2_use],
3526 MAX(op2_min, op1_min),
3527 op2_max);
3528 }
3529 } else {
3530 /* op1 > op2 */
3531 if (ssa_op->op1_use >= 0) {
3532 zend_jit_trace_set_var_range(
3533 &ssa->var_info[ssa_op->op1_use],
3534 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3535 op1_max);
3536 }
3537 if (ssa_op->op2_use >= 0) {
3538 zend_jit_trace_set_var_range(
3539 &ssa->var_info[ssa_op->op2_use],
3540 op2_min,
3541 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3542 }
3543 }
3544 break;
3545 case ZEND_IS_SMALLER:
3546 if (!exit_if_true) {
3547 /* op1 < op2 */
3548 if (ssa_op->op1_use >= 0) {
3549 zend_jit_trace_set_var_range(
3550 &ssa->var_info[ssa_op->op1_use],
3551 op1_min,
3552 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3553 }
3554 if (ssa_op->op2_use >= 0) {
3555 zend_jit_trace_set_var_range(
3556 &ssa->var_info[ssa_op->op2_use],
3557 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3558 op2_max);
3559 }
3560 } else {
3561 /* op1 >= op2 */
3562 if (ssa_op->op1_use >= 0) {
3563 zend_jit_trace_set_var_range(
3564 &ssa->var_info[ssa_op->op1_use],
3565 MAX(op1_min, op2_min),
3566 op1_max);
3567 }
3568 if (ssa_op->op2_use >= 0) {
3569 zend_jit_trace_set_var_range(
3570 &ssa->var_info[ssa_op->op2_use],
3571 op2_min,
3572 MIN(op2_max, op1_max));
3573 }
3574 }
3575 break;
3576 }
3577 }
3578
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3579 static zend_bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3580 {
3581 zend_uchar prev_opcode;
3582
3583 if (opline->op1_type == IS_CONST
3584 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3585 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3586 if (ssa_op->op2_use >= 0) {
3587 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3588 ssa_op--;
3589 opline = ssa_opcodes[ssa_op - ssa->ops];
3590 prev_opcode = opline->opcode;
3591 if (prev_opcode == ZEND_PRE_INC
3592 || prev_opcode == ZEND_PRE_DEC
3593 || prev_opcode == ZEND_POST_INC
3594 || prev_opcode == ZEND_POST_DEC) {
3595 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3596 }
3597 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3598 ssa_op--;
3599 opline = ssa_opcodes[ssa_op - ssa->ops];
3600 prev_opcode = opline->opcode;
3601 if (prev_opcode == ZEND_ADD
3602 || prev_opcode == ZEND_SUB) {
3603 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3604 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3605 }
3606 }
3607 }
3608 } else if (opline->op2_type == IS_CONST
3609 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3610 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3611 if (ssa_op->op1_use >= 0) {
3612 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3613 ssa_op--;
3614 opline = ssa_opcodes[ssa_op - ssa->ops];
3615 prev_opcode = opline->opcode;
3616 if (prev_opcode == ZEND_PRE_INC
3617 || prev_opcode == ZEND_PRE_DEC
3618 || prev_opcode == ZEND_POST_INC
3619 || prev_opcode == ZEND_POST_DEC) {
3620 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3621 }
3622 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3623 ssa_op--;
3624 opline = ssa_opcodes[ssa_op - ssa->ops];
3625 prev_opcode = opline->opcode;
3626 if (prev_opcode == ZEND_ADD
3627 || prev_opcode == ZEND_SUB) {
3628 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3629 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3630 }
3631 }
3632 }
3633 } else {
3634 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3635 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3636
3637 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3638 && prev_ssa_op != ssa->ops
3639 && prev_ssa_op->op1_use >= 0
3640 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3641 prev_ssa_op--;
3642 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3643 }
3644
3645 if (ssa_op->op1_use == prev_ssa_op->op1_use
3646 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3647 if (prev_opcode == ZEND_IS_EQUAL
3648 || prev_opcode == ZEND_IS_NOT_EQUAL
3649 || prev_opcode == ZEND_IS_SMALLER
3650 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3651 || prev_opcode == ZEND_CASE
3652 || prev_opcode == ZEND_IS_IDENTICAL
3653 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3654 || prev_opcode == ZEND_CASE_STRICT) {
3655 if (ssa_op->op1_use < 0) {
3656 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3657 return 0;
3658 }
3659 }
3660 if (ssa_op->op2_use < 0) {
3661 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3662 return 0;
3663 }
3664 }
3665 return 1;
3666 }
3667 }
3668 }
3669 return 0;
3670 }
3671
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)3672 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
3673 {
3674 const void *handler = NULL;
3675 dasm_State* dasm_state = NULL;
3676 zend_script *script = NULL;
3677 zend_lifetime_interval **ra = NULL;
3678 zend_string *name = NULL;
3679 void *checkpoint;
3680 const zend_op_array *op_array;
3681 zend_ssa *ssa, *op_array_ssa;
3682 const zend_op **ssa_opcodes;
3683 zend_jit_trace_rec *p;
3684 zend_jit_op_array_trace_extension *jit_extension;
3685 int num_op_arrays = 0;
3686 zend_jit_trace_info *t;
3687 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
3688 zend_uchar smart_branch_opcode;
3689 const void *exit_addr;
3690 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info;
3691 zend_bool send_result = 0;
3692 zend_bool skip_comparison;
3693 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
3694 zend_class_entry *ce;
3695 zend_bool ce_is_instanceof;
3696 zend_bool delayed_fetch_this = 0;
3697 zend_bool avoid_refcounting = 0;
3698 zend_bool polymorphic_side_trace =
3699 parent_trace &&
3700 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
3701 uint32_t i;
3702 zend_jit_trace_stack_frame *frame, *top, *call;
3703 zend_jit_trace_stack *stack;
3704 zend_uchar res_type = IS_UNKNOWN;
3705 const zend_op *opline, *orig_opline;
3706 const zend_ssa_op *ssa_op, *orig_ssa_op;
3707 int used_stack;
3708 uint32_t frame_flags = 0;
3709
3710 JIT_G(current_trace) = trace_buffer;
3711
3712 checkpoint = zend_arena_checkpoint(CG(arena));
3713
3714 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
3715
3716 if (!ssa) {
3717 goto jit_cleanup;
3718 }
3719
3720 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
3721 used_stack = ((zend_tssa*)ssa)->used_stack;
3722
3723 /* Register allocation */
3724 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
3725 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
3726 ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
3727 }
3728
3729 p = trace_buffer;
3730 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
3731 op_array = p->op_array;
3732 frame = JIT_G(current_frame);
3733 top = zend_jit_trace_call_frame(frame, op_array);
3734 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
3735 stack = frame->stack;
3736 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3737 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
3738 }
3739
3740 opline = p[1].opline;
3741 name = zend_jit_trace_name(op_array, opline->lineno);
3742 p += ZEND_JIT_TRACE_START_REC_SIZE;
3743
3744 dasm_init(&dasm_state, DASM_MAXSECTION);
3745 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
3746 dasm_setup(&dasm_state, dasm_actions);
3747
3748 jit_extension =
3749 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3750 op_array_ssa = &jit_extension->func_info.ssa;
3751
3752 dasm_growpc(&dasm_state, 1); /* trace needs just one global label for loop */
3753
3754 zend_jit_align_func(&dasm_state);
3755 if (!parent_trace) {
3756 zend_jit_prologue(&dasm_state);
3757 }
3758 zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM,
3759 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
3760
3761 if (!parent_trace) {
3762 zend_jit_set_last_valid_opline(opline);
3763 zend_jit_track_last_valid_opline();
3764 } else {
3765 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
3766 zend_jit_trace_opline_guard(&dasm_state, opline);
3767 } else {
3768 zend_jit_reset_last_valid_opline();
3769 }
3770 }
3771
3772 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
3773 int last_var;
3774 int parent_vars_count = 0;
3775 zend_jit_trace_stack *parent_stack = NULL;
3776
3777 if (used_stack > 0) {
3778 if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) {
3779 goto jit_failure;
3780 }
3781 }
3782
3783 if (parent_trace) {
3784 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
3785 op_array->last_var + op_array->T);
3786 if (parent_vars_count) {
3787 parent_stack =
3788 zend_jit_traces[parent_trace].stack_map +
3789 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
3790 }
3791 }
3792
3793 last_var = op_array->last_var;
3794 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3795 last_var += op_array->T;
3796 }
3797
3798 for (i = 0; i < last_var; i++) {
3799 uint32_t info = ssa->var_info[i].type;
3800
3801 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
3802 uint8_t type, mem_type;
3803
3804 type = concrete_type(info);
3805 if (i < parent_vars_count
3806 && STACK_TYPE(parent_stack, i) == type) {
3807 mem_type = STACK_MEM_TYPE(parent_stack, i);
3808 if (mem_type != IS_UNKNOWN) {
3809 SET_STACK_TYPE(stack, i, mem_type, 1);
3810 }
3811 SET_STACK_TYPE(stack, i, type, 0);
3812 } else {
3813 SET_STACK_TYPE(stack, i, type, 1);
3814 }
3815 } else if (ssa->vars[i].alias != NO_ALIAS) {
3816 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
3817 } else if (i < parent_vars_count
3818 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
3819 /* This must be already handled by trace type inference */
3820 ZEND_UNREACHABLE();
3821 // SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
3822 } else if ((info & MAY_BE_GUARD) != 0
3823 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3824 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3825 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
3826 && (opline-1)->result_type == IS_VAR
3827 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
3828 && (ssa->vars[i].use_chain != -1
3829 || (ssa->vars[i].phi_use_chain
3830 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
3831 /* Check loop-invariant variable type */
3832 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
3833 goto jit_failure;
3834 }
3835 info &= ~MAY_BE_GUARD;
3836 ssa->var_info[i].type = info;
3837 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
3838 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
3839 && op_array->function_name
3840 && i >= op_array->num_args) {
3841 /* This must be already handled by trace type inference */
3842 ZEND_UNREACHABLE();
3843 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
3844 }
3845
3846 if ((info & MAY_BE_PACKED_GUARD) != 0
3847 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3848 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3849 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
3850 && ssa->vars[i].use_chain != -1) {
3851 if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) {
3852 goto jit_failure;
3853 }
3854 info &= ~MAY_BE_PACKED_GUARD;
3855 ssa->var_info[i].type = info;
3856 }
3857 }
3858
3859 if (parent_trace) {
3860 /* Deoptimization */
3861 if (!zend_jit_trace_deoptimization(&dasm_state,
3862 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
3863 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
3864 parent_stack, parent_vars_count, ssa, stack, ra,
3865 polymorphic_side_trace)) {
3866 goto jit_failure;
3867 }
3868 }
3869
3870 if (ra
3871 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3872 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3873 for (i = 0; i < last_var; i++) {
3874 if (ra[i]
3875 && (ra[i]->flags & ZREG_LOAD) != 0
3876 && ra[i]->reg != stack[i].reg) {
3877
3878 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
3879 uint8_t op_type;
3880
3881 ssa->var_info[i].type &= ~MAY_BE_GUARD;
3882 op_type = concrete_type(ssa->var_info[i].type);
3883 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
3884 goto jit_failure;
3885 }
3886 SET_STACK_TYPE(stack, i, op_type, 1);
3887 }
3888
3889 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
3890 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
3891 goto jit_failure;
3892 }
3893 }
3894 }
3895 }
3896 }
3897
3898 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3899 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3900 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3901
3902 zend_jit_label(&dasm_state, 0); /* start of of trace loop */
3903
3904 if (ra) {
3905 zend_ssa_phi *phi = ssa->blocks[1].phis;
3906
3907 while (phi) {
3908 zend_lifetime_interval *ival = ra[phi->ssa_var];
3909
3910 if (ival) {
3911 if (ival->flags & ZREG_LOAD) {
3912 uint32_t info = ssa->var_info[phi->ssa_var].type;
3913 ZEND_ASSERT(ival->reg != ZREG_NONE);
3914
3915 if (info & MAY_BE_GUARD) {
3916 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
3917 goto jit_failure;
3918 }
3919 info &= ~MAY_BE_GUARD;
3920 ssa->var_info[phi->ssa_var].type = info;
3921 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
3922 }
3923 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_LOAD);
3924 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
3925 goto jit_failure;
3926 }
3927 } else if (ival->flags & ZREG_STORE) {
3928 ZEND_ASSERT(ival->reg != ZREG_NONE);
3929
3930 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_STORE);
3931 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg,
3932 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
3933 goto jit_failure;
3934 }
3935 } else {
3936 /* Register has to be written back on side exit */
3937 SET_STACK_REG(stack, phi->var, ival->reg);
3938 }
3939 }
3940 phi = phi->next;
3941 }
3942 }
3943
3944 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3945 // if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
3946 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
3947 //
3948 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
3949 // if (!timeout_exit_addr) {
3950 // goto jit_failure;
3951 // }
3952 // }
3953 // }
3954
3955 if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
3956 int last_var = op_array->last_var;
3957
3958 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3959 last_var += op_array->T;
3960 }
3961 for (i = 0; i < last_var; i++) {
3962 if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) {
3963 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
3964 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
3965 goto jit_failure;
3966 }
3967 }
3968 }
3969 }
3970 }
3971
3972 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
3973 for (;;p++) {
3974 if (p->op == ZEND_JIT_TRACE_VM) {
3975 uint8_t op1_type = p->op1_type;
3976 uint8_t op2_type = p->op2_type;
3977 uint8_t op3_type = p->op3_type;
3978 uint8_t orig_op1_type = op1_type;
3979 uint8_t orig_op2_type = op2_type;
3980 zend_bool op1_indirect;
3981 zend_class_entry *op1_ce = NULL;
3982 zend_class_entry *op2_ce = NULL;
3983
3984 opline = p->opline;
3985 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
3986 op1_type = IS_UNKNOWN;
3987 }
3988 if (op1_type != IS_UNKNOWN) {
3989 op1_type &= ~IS_TRACE_PACKED;
3990 }
3991 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
3992 op2_type = IS_UNKNOWN;
3993 }
3994 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
3995 op3_type = IS_UNKNOWN;
3996 }
3997
3998 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
3999 op1_ce = (zend_class_entry*)(p+1)->ce;
4000 p++;
4001 }
4002 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4003 op2_ce = (zend_class_entry*)(p+1)->ce;
4004 p++;
4005 }
4006
4007 frame_flags = 0;
4008
4009 switch (opline->opcode) {
4010 case ZEND_INIT_FCALL:
4011 case ZEND_INIT_FCALL_BY_NAME:
4012 case ZEND_INIT_NS_FCALL_BY_NAME:
4013 case ZEND_INIT_METHOD_CALL:
4014 case ZEND_INIT_DYNAMIC_CALL:
4015 case ZEND_INIT_STATIC_METHOD_CALL:
4016 case ZEND_INIT_USER_CALL:
4017 case ZEND_NEW:
4018 frame->call_level++;
4019 }
4020
4021 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4022 switch (opline->opcode) {
4023 case ZEND_PRE_INC:
4024 case ZEND_PRE_DEC:
4025 case ZEND_POST_INC:
4026 case ZEND_POST_DEC:
4027 if (opline->op1_type != IS_CV) {
4028 break;
4029 }
4030 op1_info = OP1_INFO();
4031 CHECK_OP1_TRACE_TYPE();
4032 if (!(op1_info & MAY_BE_LONG)) {
4033 break;
4034 }
4035 if (opline->result_type != IS_UNUSED) {
4036 #if USE_ABSTRACT_STACK_FOR_RES_USE_INFO
4037 res_use_info = zend_jit_trace_type_to_info(
4038 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4039 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4040 #else
4041 res_use_info = -1;
4042 if (opline->result_type == IS_CV) {
4043 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
4044
4045 if (Z_MODE(res_use_addr) != IS_REG
4046 || Z_LOAD(res_use_addr)
4047 || Z_STORE(res_use_addr)) {
4048 res_use_info = RES_USE_INFO();
4049 }
4050 }
4051 #endif
4052 res_info = RES_INFO();
4053 res_addr = RES_REG_ADDR();
4054 } else {
4055 res_use_info = -1;
4056 res_info = -1;
4057 res_addr = 0;
4058 }
4059 op1_def_info = OP1_DEF_INFO();
4060 if (op1_def_info & MAY_BE_GUARD
4061 && !has_concrete_type(op1_def_info)) {
4062 op1_def_info &= ~MAY_BE_GUARD;
4063 }
4064 if (!zend_jit_inc_dec(&dasm_state, opline,
4065 op1_info, OP1_REG_ADDR(),
4066 op1_def_info, OP1_DEF_REG_ADDR(),
4067 res_use_info, res_info,
4068 res_addr,
4069 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4070 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4071 goto jit_failure;
4072 }
4073 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4074 && !(op1_info & MAY_BE_STRING)) {
4075 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4076 if (opline->result_type != IS_UNUSED) {
4077 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4078 }
4079 }
4080 if (opline->result_type != IS_UNUSED
4081 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4082 && !(op1_info & MAY_BE_STRING)) {
4083 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4084 }
4085 goto done;
4086 case ZEND_BW_OR:
4087 case ZEND_BW_AND:
4088 case ZEND_BW_XOR:
4089 case ZEND_SL:
4090 case ZEND_SR:
4091 case ZEND_MOD:
4092 op1_info = OP1_INFO();
4093 CHECK_OP1_TRACE_TYPE();
4094 op2_info = OP2_INFO();
4095 CHECK_OP2_TRACE_TYPE();
4096 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4097 break;
4098 }
4099 if (!(op1_info & MAY_BE_LONG)
4100 || !(op2_info & MAY_BE_LONG)) {
4101 break;
4102 }
4103 res_addr = RES_REG_ADDR();
4104 if (Z_MODE(res_addr) != IS_REG
4105 && opline->result_type == IS_TMP_VAR
4106 && (p+1)->op == ZEND_JIT_TRACE_VM
4107 && (p+1)->opline == opline + 1
4108 && ((opline+1)->opcode == ZEND_SEND_VAL
4109 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
4110 && frame
4111 && frame->call
4112 && frame->call->func
4113 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
4114 && (opline+1)->op1_type == IS_TMP_VAR
4115 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
4116 && (opline+1)->op1.var == opline->result.var) {
4117 p++;
4118 if (frame->call) {
4119 uint8_t res_type = p->op1_type;
4120 if (res_type & IS_TRACE_REFERENCE) {
4121 res_type = IS_UNKNOWN;
4122 }
4123 if (res_type != IS_UNKNOWN) {
4124 zend_jit_trace_send_type(opline+1, frame->call, res_type);
4125 }
4126 }
4127 while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE ||
4128 (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4129 p++;
4130 }
4131 send_result = 1;
4132 res_use_info = -1;
4133 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4134 if (!zend_jit_reuse_ip(&dasm_state)) {
4135 goto jit_failure;
4136 }
4137 } else {
4138 #if USE_ABSTRACT_STACK_FOR_RES_USE_INFO
4139 res_use_info = zend_jit_trace_type_to_info(
4140 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4141 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4142 #else
4143 res_use_info = -1;
4144 if (opline->result_type == IS_CV) {
4145 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
4146
4147 if (Z_MODE(res_use_addr) != IS_REG
4148 || Z_LOAD(res_use_addr)
4149 || Z_STORE(res_use_addr)) {
4150 res_use_info = RES_USE_INFO();
4151 }
4152 }
4153 #endif
4154 }
4155 res_info = RES_INFO();
4156 if (!zend_jit_long_math(&dasm_state, opline,
4157 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4158 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4159 res_use_info, res_info, res_addr,
4160 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4161 goto jit_failure;
4162 }
4163 goto done;
4164 case ZEND_ADD:
4165 case ZEND_SUB:
4166 case ZEND_MUL:
4167 // case ZEND_DIV: // TODO: check for division by zero ???
4168 op1_info = OP1_INFO();
4169 CHECK_OP1_TRACE_TYPE();
4170 op2_info = OP2_INFO();
4171 CHECK_OP2_TRACE_TYPE();
4172 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4173 break;
4174 }
4175 if (opline->opcode == ZEND_ADD &&
4176 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4177 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4178 /* pass */
4179 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4180 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4181 break;
4182 }
4183 res_addr = RES_REG_ADDR();
4184 if (Z_MODE(res_addr) != IS_REG
4185 && opline->result_type == IS_TMP_VAR
4186 && (p+1)->op == ZEND_JIT_TRACE_VM
4187 && (p+1)->opline == opline + 1
4188 && ((opline+1)->opcode == ZEND_SEND_VAL
4189 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
4190 && frame
4191 && frame->call
4192 && frame->call->func
4193 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
4194 && (opline+1)->op1_type == IS_TMP_VAR
4195 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
4196 && (opline+1)->op1.var == opline->result.var) {
4197 p++;
4198 if (frame->call
4199 && frame->call->func
4200 && frame->call->func->type == ZEND_USER_FUNCTION) {
4201 uint8_t res_type = p->op1_type;
4202 if (res_type & IS_TRACE_REFERENCE) {
4203 res_type = IS_UNKNOWN;
4204 }
4205 if (res_type != IS_UNKNOWN) {
4206 zend_jit_trace_send_type(opline+1, frame->call, res_type);
4207 }
4208 }
4209 while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE ||
4210 (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4211 p++;
4212 }
4213 send_result = 1;
4214 res_use_info = -1;
4215 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4216 if (!zend_jit_reuse_ip(&dasm_state)) {
4217 goto jit_failure;
4218 }
4219 } else {
4220 #if USE_ABSTRACT_STACK_FOR_RES_USE_INFO
4221 res_use_info = zend_jit_trace_type_to_info(
4222 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4223 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4224 #else
4225 res_use_info = -1;
4226 if (opline->result_type == IS_CV) {
4227 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
4228
4229 if (Z_MODE(res_use_addr) != IS_REG
4230 || Z_LOAD(res_use_addr)
4231 || Z_STORE(res_use_addr)) {
4232 res_use_info = RES_USE_INFO();
4233 }
4234 }
4235 #endif
4236 }
4237 res_info = RES_INFO();
4238 if (opline->opcode == ZEND_ADD &&
4239 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4240 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4241 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, op2_info, res_addr)) {
4242 goto jit_failure;
4243 }
4244 } else {
4245 if (!zend_jit_math(&dasm_state, opline,
4246 op1_info, OP1_REG_ADDR(),
4247 op2_info, OP2_REG_ADDR(),
4248 res_use_info, res_info, res_addr,
4249 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4250 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4251 goto jit_failure;
4252 }
4253 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4254 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4255 && has_concrete_type(op1_info)
4256 && has_concrete_type(op2_info)) {
4257 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4258 }
4259 }
4260 goto done;
4261 case ZEND_CONCAT:
4262 case ZEND_FAST_CONCAT:
4263 op1_info = OP1_INFO();
4264 CHECK_OP1_TRACE_TYPE();
4265 op2_info = OP2_INFO();
4266 CHECK_OP2_TRACE_TYPE();
4267 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4268 break;
4269 }
4270 if (!(op1_info & MAY_BE_STRING) ||
4271 !(op2_info & MAY_BE_STRING)) {
4272 break;
4273 }
4274 res_addr = RES_REG_ADDR();
4275 if (Z_MODE(res_addr) != IS_REG
4276 && opline->result_type == IS_TMP_VAR
4277 && (p+1)->op == ZEND_JIT_TRACE_VM
4278 && (p+1)->opline == opline + 1
4279 && ((opline+1)->opcode == ZEND_SEND_VAL
4280 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
4281 && frame
4282 && frame->call
4283 && frame->call->func
4284 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
4285 && (opline+1)->op1_type == IS_TMP_VAR
4286 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
4287 && (opline+1)->op1.var == opline->result.var) {
4288 p++;
4289 if (frame->call
4290 && frame->call->func
4291 && frame->call->func->type == ZEND_USER_FUNCTION) {
4292 uint8_t res_type = p->op1_type;
4293 if (res_type & IS_TRACE_REFERENCE) {
4294 res_type = IS_UNKNOWN;
4295 }
4296 if (res_type != IS_UNKNOWN) {
4297 zend_jit_trace_send_type(opline+1, frame->call, res_type);
4298 }
4299 }
4300 while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE ||
4301 (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4302 p++;
4303 }
4304 send_result = 1;
4305 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4306 if (!zend_jit_reuse_ip(&dasm_state)) {
4307 goto jit_failure;
4308 }
4309 }
4310 if (!zend_jit_concat(&dasm_state, opline,
4311 op1_info, op2_info, res_addr,
4312 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4313 goto jit_failure;
4314 }
4315 goto done;
4316 case ZEND_ASSIGN_OP:
4317 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4318 break;
4319 }
4320 op1_info = OP1_INFO();
4321 CHECK_OP1_TRACE_TYPE();
4322 op2_info = OP2_INFO();
4323 CHECK_OP2_TRACE_TYPE();
4324 if (!zend_jit_supported_binary_op(
4325 opline->extended_value, op1_info, op2_info)) {
4326 break;
4327 }
4328 op1_def_info = OP1_DEF_INFO();
4329 if (op1_def_info & MAY_BE_GUARD
4330 && !has_concrete_type(op1_def_info)) {
4331 op1_def_info &= ~MAY_BE_GUARD;
4332 }
4333 if (!zend_jit_assign_op(&dasm_state, opline,
4334 op1_info, op1_def_info, OP1_RANGE(),
4335 op2_info, OP2_RANGE(),
4336 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4337 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4338 goto jit_failure;
4339 }
4340 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4341 && has_concrete_type(op1_info)
4342 && has_concrete_type(op2_info)) {
4343 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4344 if (opline->result_type != IS_UNUSED) {
4345 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4346 }
4347 }
4348 goto done;
4349 case ZEND_ASSIGN_DIM_OP:
4350 if (opline->result_type != IS_UNUSED) {
4351 break;
4352 }
4353 if (!zend_jit_supported_binary_op(
4354 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4355 break;
4356 }
4357 if (opline->op1_type == IS_CV
4358 && (opline+1)->op1_type == IS_CV
4359 && (opline+1)->op1.var == opline->op1.var) {
4360 /* skip $a[x] += $a; */
4361 break;
4362 }
4363 op1_info = OP1_INFO();
4364 op1_addr = OP1_REG_ADDR();
4365 if (opline->op1_type == IS_VAR) {
4366 if (orig_op1_type != IS_UNKNOWN
4367 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4368 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4369 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4370 goto jit_failure;
4371 }
4372 } else {
4373 break;
4374 }
4375 }
4376 if (orig_op1_type != IS_UNKNOWN
4377 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4378 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4379 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4380 goto jit_failure;
4381 }
4382 if (opline->op1_type == IS_CV
4383 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4384 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4385 }
4386 } else {
4387 CHECK_OP1_TRACE_TYPE();
4388 }
4389 op2_info = OP2_INFO();
4390 CHECK_OP2_TRACE_TYPE();
4391 op1_data_info = OP1_DATA_INFO();
4392 CHECK_OP1_DATA_TRACE_TYPE();
4393 op1_def_info = OP1_DEF_INFO();
4394 if (!zend_jit_assign_dim_op(&dasm_state, opline,
4395 op1_info, op1_def_info, op1_addr, op2_info,
4396 op1_data_info, OP1_DATA_RANGE(),
4397 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4398 goto jit_failure;
4399 }
4400 goto done;
4401 case ZEND_PRE_INC_OBJ:
4402 case ZEND_PRE_DEC_OBJ:
4403 case ZEND_POST_INC_OBJ:
4404 case ZEND_POST_DEC_OBJ:
4405 if (opline->op2_type != IS_CONST
4406 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4407 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4408 break;
4409 }
4410 ce = NULL;
4411 ce_is_instanceof = 0;
4412 delayed_fetch_this = 0;
4413 op1_indirect = 0;
4414 if (opline->op1_type == IS_UNUSED) {
4415 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4416 ce = op_array->scope;
4417 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4418 op1_addr = 0;
4419 } else {
4420 if (ssa_op->op1_use >= 0) {
4421 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4422 }
4423 op1_info = OP1_INFO();
4424 if (!(op1_info & MAY_BE_OBJECT)) {
4425 break;
4426 }
4427 op1_addr = OP1_REG_ADDR();
4428 if (opline->op1_type == IS_VAR) {
4429 if (orig_op1_type != IS_UNKNOWN
4430 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4431 op1_indirect = 1;
4432 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4433 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4434 goto jit_failure;
4435 }
4436 }
4437 }
4438 if (orig_op1_type != IS_UNKNOWN
4439 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4440 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4441 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4442 goto jit_failure;
4443 }
4444 if (opline->op1_type == IS_CV
4445 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4446 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4447 }
4448 } else {
4449 CHECK_OP1_TRACE_TYPE();
4450 }
4451 if (!(op1_info & MAY_BE_OBJECT)) {
4452 break;
4453 }
4454 if (ssa->var_info && ssa->ops) {
4455 if (ssa_op->op1_use >= 0) {
4456 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4457 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4458 ce = op1_ssa->ce;
4459 ce_is_instanceof = op1_ssa->is_instanceof;
4460 }
4461 }
4462 }
4463 }
4464 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4465 op1_info, op1_addr,
4466 op1_indirect, ce, ce_is_instanceof, delayed_fetch_this, op1_ce,
4467 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4468 goto jit_failure;
4469 }
4470 goto done;
4471 case ZEND_ASSIGN_OBJ_OP:
4472 if (opline->result_type != IS_UNUSED) {
4473 break;
4474 }
4475 if (opline->op2_type != IS_CONST
4476 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4477 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4478 break;
4479 }
4480 if (opline->op1_type == IS_CV
4481 && (opline+1)->op1_type == IS_CV
4482 && (opline+1)->op1.var == opline->op1.var) {
4483 /* skip $a->prop += $a; */
4484 break;
4485 }
4486 if (!zend_jit_supported_binary_op(
4487 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4488 break;
4489 }
4490 ce = NULL;
4491 ce_is_instanceof = 0;
4492 delayed_fetch_this = 0;
4493 op1_indirect = 0;
4494 if (opline->op1_type == IS_UNUSED) {
4495 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4496 ce = op_array->scope;
4497 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4498 op1_addr = 0;
4499 } else {
4500 if (ssa_op->op1_use >= 0) {
4501 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4502 }
4503 op1_info = OP1_INFO();
4504 if (!(op1_info & MAY_BE_OBJECT)) {
4505 break;
4506 }
4507 op1_addr = OP1_REG_ADDR();
4508 if (opline->op1_type == IS_VAR) {
4509 if (orig_op1_type != IS_UNKNOWN
4510 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4511 op1_indirect = 1;
4512 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4513 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4514 goto jit_failure;
4515 }
4516 }
4517 }
4518 if (orig_op1_type != IS_UNKNOWN
4519 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4520 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4521 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4522 goto jit_failure;
4523 }
4524 if (opline->op1_type == IS_CV
4525 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4526 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4527 }
4528 } else {
4529 CHECK_OP1_TRACE_TYPE();
4530 }
4531 if (!(op1_info & MAY_BE_OBJECT)) {
4532 break;
4533 }
4534 if (ssa->var_info && ssa->ops) {
4535 if (ssa_op->op1_use >= 0) {
4536 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4537 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4538 ce = op1_ssa->ce;
4539 ce_is_instanceof = op1_ssa->is_instanceof;
4540 }
4541 }
4542 }
4543 }
4544 op1_data_info = OP1_DATA_INFO();
4545 CHECK_OP1_DATA_TRACE_TYPE();
4546 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
4547 op1_info, op1_addr, op1_data_info, OP1_DATA_RANGE(),
4548 op1_indirect, ce, ce_is_instanceof, delayed_fetch_this, op1_ce,
4549 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4550 goto jit_failure;
4551 }
4552 goto done;
4553 case ZEND_ASSIGN_OBJ:
4554 if (opline->op2_type != IS_CONST
4555 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4556 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4557 break;
4558 }
4559 ce = NULL;
4560 ce_is_instanceof = 0;
4561 delayed_fetch_this = 0;
4562 op1_indirect = 0;
4563 if (opline->op1_type == IS_UNUSED) {
4564 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4565 ce = op_array->scope;
4566 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4567 op1_addr = 0;
4568 } else {
4569 if (ssa_op->op1_use >= 0) {
4570 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4571 }
4572 op1_info = OP1_INFO();
4573 if (!(op1_info & MAY_BE_OBJECT)) {
4574 break;
4575 }
4576 op1_addr = OP1_REG_ADDR();
4577 if (opline->op1_type == IS_VAR) {
4578 if (orig_op1_type != IS_UNKNOWN
4579 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4580 op1_indirect = 1;
4581 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4582 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4583 goto jit_failure;
4584 }
4585 }
4586 }
4587 if (orig_op1_type != IS_UNKNOWN
4588 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4589 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4590 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4591 goto jit_failure;
4592 }
4593 if (opline->op1_type == IS_CV
4594 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4595 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4596 }
4597 } else {
4598 CHECK_OP1_TRACE_TYPE();
4599 }
4600 if (!(op1_info & MAY_BE_OBJECT)) {
4601 break;
4602 }
4603 if (ssa->var_info && ssa->ops) {
4604 if (ssa_op->op1_use >= 0) {
4605 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4606 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4607 ce = op1_ssa->ce;
4608 ce_is_instanceof = op1_ssa->is_instanceof;
4609 }
4610 }
4611 }
4612 }
4613 op1_data_info = OP1_DATA_INFO();
4614 CHECK_OP1_DATA_TRACE_TYPE();
4615 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4616 op1_info, op1_addr, op1_data_info,
4617 op1_indirect, ce, ce_is_instanceof, delayed_fetch_this, op1_ce,
4618 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4619 goto jit_failure;
4620 }
4621 if ((opline+1)->op1_type == IS_CV
4622 && (ssa_op+1)->op1_def >= 0
4623 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4624 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4625 }
4626 goto done;
4627 case ZEND_ASSIGN_DIM:
4628 op1_info = OP1_INFO();
4629 op1_addr = OP1_REG_ADDR();
4630 if (opline->op1_type == IS_CV
4631 && (opline+1)->op1_type == IS_CV
4632 && (opline+1)->op1.var == opline->op1.var) {
4633 /* skip $a[x] = $a; */
4634 break;
4635 }
4636 if (opline->op1_type == IS_VAR) {
4637 if (orig_op1_type != IS_UNKNOWN
4638 && (orig_op1_type & IS_TRACE_INDIRECT)
4639 && opline->result_type == IS_UNUSED) {
4640 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4641 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4642 goto jit_failure;
4643 }
4644 } else {
4645 break;
4646 }
4647 }
4648 if (orig_op1_type != IS_UNKNOWN
4649 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4650 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4651 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4652 goto jit_failure;
4653 }
4654 if (opline->op1_type == IS_CV
4655 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4656 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4657 }
4658 } else {
4659 CHECK_OP1_TRACE_TYPE();
4660 }
4661 op2_info = OP2_INFO();
4662 CHECK_OP2_TRACE_TYPE();
4663 op1_data_info = OP1_DATA_INFO();
4664 CHECK_OP1_DATA_TRACE_TYPE();
4665 if (!zend_jit_assign_dim(&dasm_state, opline,
4666 op1_info, op1_addr, op2_info, op1_data_info,
4667 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4668 goto jit_failure;
4669 }
4670 if ((opline+1)->op1_type == IS_CV
4671 && (ssa_op+1)->op1_def >= 0
4672 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4673 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4674 }
4675 goto done;
4676 case ZEND_ASSIGN:
4677 if (opline->op1_type != IS_CV) {
4678 break;
4679 }
4680 op2_addr = OP2_REG_ADDR();
4681 op2_info = OP2_INFO();
4682 if (ra
4683 && ssa_op->op2_def >= 0
4684 && (!ssa->vars[ssa_op->op2_def].no_val
4685 || (zend_jit_trace_type_to_info(STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var))) & MAY_BE_ANY) !=
4686 (op2_info & MAY_BE_ANY))) {
4687 op2_def_addr = OP2_DEF_REG_ADDR();
4688 } else {
4689 op2_def_addr = op2_addr;
4690 }
4691 CHECK_OP2_TRACE_TYPE();
4692 op1_info = OP1_INFO();
4693 CHECK_OP1_TRACE_TYPE();
4694 op1_def_info = OP1_DEF_INFO();
4695 op1_addr = OP1_REG_ADDR();
4696 op1_def_addr = OP1_DEF_REG_ADDR();
4697 if (Z_MODE(op1_def_addr) != IS_REG &&
4698 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
4699 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
4700 /* type may be not set */
4701 op1_info |= MAY_BE_NULL;
4702 }
4703 if (orig_op1_type != IS_UNKNOWN) {
4704 if (orig_op1_type & IS_TRACE_REFERENCE) {
4705 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4706 !ssa->var_info[ssa_op->op1_use].guarded_reference, 0)) {
4707 goto jit_failure;
4708 }
4709 if (opline->op1_type == IS_CV
4710 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4711 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4712 }
4713 if (opline->result_type == IS_UNUSED) {
4714 res_addr = 0;
4715 } else {
4716 res_addr = RES_REG_ADDR();
4717 }
4718 if (!zend_jit_assign_to_typed_ref(&dasm_state, opline, opline->op2_type, op2_addr, res_addr, 1)) {
4719 goto jit_failure;
4720 }
4721 op1_def_addr = op1_addr;
4722 op1_def_info &= ~MAY_BE_REF;
4723 } else if (op1_info & MAY_BE_REF) {
4724 if (!zend_jit_noref_guard(&dasm_state, opline, op1_addr)) {
4725 goto jit_failure;
4726 }
4727 op1_info &= ~MAY_BE_REF;
4728 op1_def_info &= ~MAY_BE_REF;
4729 }
4730 }
4731 if (opline->result_type == IS_UNUSED) {
4732 res_addr = 0;
4733 res_info = -1;
4734 } else {
4735 res_addr = RES_REG_ADDR();
4736 res_info = RES_INFO();
4737 if (Z_MODE(res_addr) != IS_REG
4738 && opline->result_type == IS_TMP_VAR
4739 && (p+1)->op == ZEND_JIT_TRACE_VM
4740 && (p+1)->opline == opline + 1
4741 && ((opline+1)->opcode == ZEND_SEND_VAL
4742 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
4743 && frame
4744 && frame->call
4745 && frame->call->func
4746 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
4747 && (opline+1)->op1_type == IS_TMP_VAR
4748 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
4749 && (opline+1)->op1.var == opline->result.var) {
4750 p++;
4751 if (frame->call
4752 && frame->call->func
4753 && frame->call->func->type == ZEND_USER_FUNCTION) {
4754 uint8_t res_type = p->op1_type;
4755 if (res_type & IS_TRACE_REFERENCE) {
4756 res_type = IS_UNKNOWN;
4757 }
4758 if (res_type != IS_UNKNOWN) {
4759 zend_jit_trace_send_type(opline+1, frame->call, res_type);
4760 }
4761 }
4762 while ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE ||
4763 (p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4764 p++;
4765 }
4766 send_result = 1;
4767 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4768 if (!zend_jit_reuse_ip(&dasm_state)) {
4769 goto jit_failure;
4770 }
4771 }
4772 }
4773 if (!zend_jit_assign(&dasm_state, opline,
4774 op1_info, op1_addr,
4775 op1_def_info, op1_def_addr,
4776 op2_info, op2_addr, op2_def_addr,
4777 res_info, res_addr,
4778 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4779 goto jit_failure;
4780 }
4781 if (opline->op2_type == IS_CV
4782 && ssa_op->op2_def >= 0
4783 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
4784 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
4785 }
4786 goto done;
4787 case ZEND_CAST:
4788 if (opline->extended_value != op1_type) {
4789 break;
4790 }
4791 /* break missing intentionally */
4792 case ZEND_QM_ASSIGN:
4793 op1_addr = OP1_REG_ADDR();
4794 if (ra
4795 && ssa_op->op1_def >= 0
4796 && !ssa->vars[ssa_op->op1_def].no_val) {
4797 op1_def_addr = OP1_DEF_REG_ADDR();
4798 } else {
4799 op1_def_addr = op1_addr;
4800 }
4801 op1_info = OP1_INFO();
4802 CHECK_OP1_TRACE_TYPE();
4803 res_info = RES_INFO();
4804 #if USE_ABSTRACT_STACK_FOR_RES_USE_INFO
4805 res_use_info = zend_jit_trace_type_to_info(
4806 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4807 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4808 #else
4809 res_use_info = MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE;
4810 #endif
4811 res_addr = RES_REG_ADDR();
4812 if (Z_MODE(res_addr) != IS_REG &&
4813 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
4814 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
4815 /* type may be not set */
4816 res_use_info |= MAY_BE_NULL;
4817 }
4818 if (!zend_jit_qm_assign(&dasm_state, opline,
4819 op1_info, op1_addr, op1_def_addr,
4820 res_use_info, res_info, res_addr)) {
4821 goto jit_failure;
4822 }
4823 if (opline->op1_type == IS_CV
4824 && ssa_op->op1_def >= 0
4825 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4826 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
4827 }
4828 goto done;
4829 case ZEND_INIT_FCALL:
4830 case ZEND_INIT_FCALL_BY_NAME:
4831 case ZEND_INIT_NS_FCALL_BY_NAME:
4832 frame_flags = TRACE_FRAME_MASK_NESTED;
4833 if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, used_stack < 0)) {
4834 goto jit_failure;
4835 }
4836 goto done;
4837 case ZEND_SEND_VAL:
4838 case ZEND_SEND_VAL_EX:
4839 if (opline->op2_type == IS_CONST) {
4840 /* Named parameters not supported in JIT */
4841 break;
4842 }
4843 if (opline->opcode == ZEND_SEND_VAL_EX
4844 && opline->op2.num > MAX_ARG_FLAG_NUM) {
4845 break;
4846 }
4847 op1_info = OP1_INFO();
4848 CHECK_OP1_TRACE_TYPE();
4849 if (!zend_jit_send_val(&dasm_state, opline,
4850 op1_info, OP1_REG_ADDR())) {
4851 goto jit_failure;
4852 }
4853 if (frame->call
4854 && frame->call->func
4855 && frame->call->func->type == ZEND_USER_FUNCTION) {
4856 if (opline->op1_type == IS_CONST) {
4857 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
4858 } else if (op1_type != IS_UNKNOWN) {
4859 if (op1_type == IS_UNDEF) {
4860 op1_type = IS_NULL;
4861 }
4862 zend_jit_trace_send_type(opline, frame->call, op1_type);
4863 }
4864 }
4865 goto done;
4866 case ZEND_SEND_REF:
4867 if (opline->op2_type == IS_CONST) {
4868 /* Named parameters not supported in JIT */
4869 break;
4870 }
4871 op1_info = OP1_INFO();
4872 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
4873 op1_info, 0)) {
4874 goto jit_failure;
4875 }
4876 if (opline->op1_type == IS_CV
4877 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4878 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4879 }
4880 goto done;
4881 case ZEND_SEND_VAR:
4882 case ZEND_SEND_VAR_EX:
4883 case ZEND_SEND_VAR_NO_REF:
4884 case ZEND_SEND_VAR_NO_REF_EX:
4885 case ZEND_SEND_FUNC_ARG:
4886 if (opline->op2_type == IS_CONST) {
4887 /* Named parameters not supported in JIT */
4888 break;
4889 }
4890 if ((opline->opcode == ZEND_SEND_VAR_EX
4891 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
4892 && opline->op2.num > MAX_ARG_FLAG_NUM) {
4893 break;
4894 }
4895 op1_addr = OP1_REG_ADDR();
4896 if (ra
4897 && ssa_op->op1_def >= 0
4898 && !ssa->vars[ssa_op->op1_def].no_val) {
4899 op1_def_addr = OP1_DEF_REG_ADDR();
4900 } else {
4901 op1_def_addr = op1_addr;
4902 }
4903 op1_info = OP1_INFO();
4904 CHECK_OP1_TRACE_TYPE();
4905 if (!zend_jit_send_var(&dasm_state, opline, op_array,
4906 op1_info, op1_addr, op1_def_addr)) {
4907 goto jit_failure;
4908 }
4909 if (opline->op1_type == IS_CV
4910 && ssa_op->op1_def >= 0
4911 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4912 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
4913 }
4914 if (frame->call
4915 && frame->call->func
4916 && frame->call->func->type == ZEND_USER_FUNCTION) {
4917 if ((opline->opcode == ZEND_SEND_VAR_EX
4918 || opline->opcode == ZEND_SEND_FUNC_ARG)
4919 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
4920 goto done;
4921 }
4922 if (op1_type != IS_UNKNOWN) {
4923 if (op1_type == IS_UNDEF) {
4924 op1_type = IS_NULL;
4925 }
4926 zend_jit_trace_send_type(opline, frame->call, op1_type);
4927 }
4928 }
4929 goto done;
4930 case ZEND_CHECK_FUNC_ARG:
4931 if (!JIT_G(current_frame)
4932 || !JIT_G(current_frame)->call
4933 || !JIT_G(current_frame)->call->func) {
4934 break;
4935 }
4936 if (opline->op2_type == IS_CONST
4937 || opline->op2.num > MAX_ARG_FLAG_NUM) {
4938 /* Named parameters not supported in JIT */
4939 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
4940 break;
4941 }
4942 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
4943 goto jit_failure;
4944 }
4945 goto done;
4946 case ZEND_CHECK_UNDEF_ARGS:
4947 if (JIT_G(current_frame)
4948 && JIT_G(current_frame)->call) {
4949 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
4950 }
4951 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
4952 goto jit_failure;
4953 }
4954 goto done;
4955 case ZEND_DO_UCALL:
4956 case ZEND_DO_ICALL:
4957 case ZEND_DO_FCALL_BY_NAME:
4958 case ZEND_DO_FCALL:
4959 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
4960 goto jit_failure;
4961 }
4962 goto done;
4963 case ZEND_IS_EQUAL:
4964 case ZEND_IS_NOT_EQUAL:
4965 case ZEND_IS_SMALLER:
4966 case ZEND_IS_SMALLER_OR_EQUAL:
4967 case ZEND_CASE:
4968 op1_info = OP1_INFO();
4969 op2_info = OP2_INFO();
4970 skip_comparison =
4971 ssa_op != ssa->ops &&
4972 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
4973 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
4974 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
4975 CHECK_OP1_TRACE_TYPE();
4976 CHECK_OP2_TRACE_TYPE();
4977 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4978 zend_bool exit_if_true = 0;
4979 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
4980 uint32_t exit_point;
4981
4982 if (ra) {
4983 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
4984 }
4985 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
4986 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4987 if (!exit_addr) {
4988 goto jit_failure;
4989 }
4990 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
4991 if (!zend_jit_cmp(&dasm_state, opline,
4992 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4993 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4994 RES_REG_ADDR(),
4995 zend_may_throw(opline, ssa_op, op_array, ssa),
4996 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
4997 goto jit_failure;
4998 }
4999 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5000 } else {
5001 smart_branch_opcode = 0;
5002 exit_addr = NULL;
5003 if (!zend_jit_cmp(&dasm_state, opline,
5004 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5005 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5006 RES_REG_ADDR(),
5007 zend_may_throw(opline, ssa_op, op_array, ssa),
5008 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5009 goto jit_failure;
5010 }
5011 }
5012 goto done;
5013 case ZEND_IS_IDENTICAL:
5014 case ZEND_IS_NOT_IDENTICAL:
5015 case ZEND_CASE_STRICT:
5016 op1_info = OP1_INFO();
5017 op2_info = OP2_INFO();
5018 skip_comparison =
5019 ssa_op != ssa->ops &&
5020 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5021 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5022 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5023 CHECK_OP1_TRACE_TYPE();
5024 CHECK_OP2_TRACE_TYPE();
5025 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5026 zend_bool exit_if_true = 0;
5027 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5028 uint32_t exit_point;
5029
5030 if (ra) {
5031 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5032 }
5033 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5034 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5035 if (!exit_addr) {
5036 goto jit_failure;
5037 }
5038 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5039 exit_if_true = !exit_if_true;
5040 }
5041 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5042 if (!zend_jit_identical(&dasm_state, opline,
5043 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5044 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5045 RES_REG_ADDR(),
5046 zend_may_throw(opline, ssa_op, op_array, ssa),
5047 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5048 goto jit_failure;
5049 }
5050 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5051 } else {
5052 smart_branch_opcode = 0;
5053 exit_addr = NULL;
5054 if (!zend_jit_identical(&dasm_state, opline,
5055 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5056 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5057 RES_REG_ADDR(),
5058 zend_may_throw(opline, ssa_op, op_array, ssa),
5059 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5060 goto jit_failure;
5061 }
5062 }
5063 goto done;
5064 case ZEND_DEFINED:
5065 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5066 zend_bool exit_if_true = 0;
5067 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5068 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5069
5070 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5071 if (!exit_addr) {
5072 goto jit_failure;
5073 }
5074 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5075 } else {
5076 smart_branch_opcode = 0;
5077 exit_addr = NULL;
5078 }
5079 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5080 goto jit_failure;
5081 }
5082 goto done;
5083 case ZEND_TYPE_CHECK:
5084 if (opline->extended_value == MAY_BE_RESOURCE) {
5085 // TODO: support for is_resource() ???
5086 break;
5087 }
5088 op1_info = OP1_INFO();
5089 CHECK_OP1_TRACE_TYPE();
5090 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5091 zend_bool exit_if_true = 0;
5092 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5093 uint32_t exit_point;
5094
5095 if (ra) {
5096 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5097 }
5098 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5099 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5100 if (!exit_addr) {
5101 goto jit_failure;
5102 }
5103 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5104 } else {
5105 smart_branch_opcode = 0;
5106 exit_addr = NULL;
5107 }
5108 if (!zend_jit_type_check(&dasm_state, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5109 goto jit_failure;
5110 }
5111 goto done;
5112 case ZEND_RETURN:
5113 op1_info = OP1_INFO();
5114 CHECK_OP1_TRACE_TYPE();
5115 if (opline->op1_type == IS_CONST) {
5116 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5117 } else if (op1_type != IS_UNKNOWN) {
5118 res_type = op1_type;
5119 }
5120 if (op_array->type == ZEND_EVAL_CODE
5121 // TODO: support for top-level code
5122 || !op_array->function_name
5123 // TODO: support for IS_UNDEF ???
5124 || (op1_info & MAY_BE_UNDEF)) {
5125 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5126 goto jit_failure;
5127 }
5128 } else {
5129 int j;
5130 int may_throw = 0;
5131 zend_bool left_frame = 0;
5132
5133 if (!zend_jit_return(&dasm_state, opline, op_array,
5134 op1_info, OP1_REG_ADDR())) {
5135 goto jit_failure;
5136 }
5137 for (j = 0 ; j < op_array->last_var; j++) {
5138 uint32_t info;
5139 zend_uchar type;
5140
5141 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5142 type = STACK_TYPE(stack, j);
5143 info = zend_jit_trace_type_to_info_ex(type, info);
5144 if (opline->op1_type == IS_CV
5145 && EX_VAR_TO_NUM(opline->op1.var) == j
5146 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5147 if (JIT_G(current_frame)
5148 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5149 continue;
5150 } else {
5151 info |= MAY_BE_NULL;
5152 }
5153 }
5154 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5155 if (!left_frame) {
5156 left_frame = 1;
5157 if (!zend_jit_leave_frame(&dasm_state)) {
5158 goto jit_failure;
5159 }
5160 }
5161 if (!zend_jit_free_cv(&dasm_state, info, j)) {
5162 goto jit_failure;
5163 }
5164 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5165 if (info & MAY_BE_RC1) {
5166 may_throw = 1;
5167 }
5168 }
5169 }
5170 }
5171 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
5172 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5173 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5174 goto jit_failure;
5175 }
5176 }
5177 goto done;
5178 case ZEND_BOOL:
5179 case ZEND_BOOL_NOT:
5180 op1_info = OP1_INFO();
5181 CHECK_OP1_TRACE_TYPE();
5182 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5183 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5184 -1, -1,
5185 zend_may_throw(opline, ssa_op, op_array, ssa),
5186 opline->opcode, NULL)) {
5187 goto jit_failure;
5188 }
5189 goto done;
5190 case ZEND_JMPZ:
5191 case ZEND_JMPNZ:
5192 case ZEND_JMPZNZ:
5193 case ZEND_JMPZ_EX:
5194 case ZEND_JMPNZ_EX:
5195 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5196 const zend_op *exit_opline = NULL;
5197 uint32_t exit_point;
5198
5199 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5200 /* taken branch */
5201 if (opline->opcode == ZEND_JMPNZ_EX) {
5202 smart_branch_opcode = ZEND_JMPZ_EX;
5203 } else if (opline->opcode == ZEND_JMPZ_EX) {
5204 smart_branch_opcode = ZEND_JMPNZ_EX;
5205 } else if (opline->opcode == ZEND_JMPNZ) {
5206 smart_branch_opcode = ZEND_JMPZ;
5207 } else {
5208 smart_branch_opcode = ZEND_JMPNZ;
5209 }
5210 exit_opline = (opline->opcode == ZEND_JMPZNZ) ?
5211 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
5212 opline + 1;
5213 } else if (opline->opcode == ZEND_JMPZNZ) {
5214 ZEND_ASSERT((p+1)->opline == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value));
5215 smart_branch_opcode = ZEND_JMPZ;
5216 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5217 } else if ((p+1)->opline == opline + 1) {
5218 /* not taken branch */
5219 smart_branch_opcode = opline->opcode;
5220 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5221 } else {
5222 ZEND_UNREACHABLE();
5223 }
5224 if (ra) {
5225 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5226 }
5227 if (opline->result_type == IS_TMP_VAR) {
5228 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5229 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5230
5231 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5232 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5233 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5234 } else {
5235 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5236 }
5237 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5238 if (!exit_addr) {
5239 goto jit_failure;
5240 }
5241 } else {
5242 ZEND_UNREACHABLE();
5243 }
5244 if (opline->result_type == IS_UNDEF) {
5245 res_addr = 0;
5246 } else {
5247 res_addr = RES_REG_ADDR();
5248 }
5249 op1_info = OP1_INFO();
5250 CHECK_OP1_TRACE_TYPE();
5251 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5252 op1_info, OP1_REG_ADDR(), res_addr,
5253 -1, -1,
5254 zend_may_throw(opline, ssa_op, op_array, ssa),
5255 smart_branch_opcode, exit_addr)) {
5256 goto jit_failure;
5257 }
5258 goto done;
5259 case ZEND_ISSET_ISEMPTY_CV:
5260 if ((opline->extended_value & ZEND_ISEMPTY)) {
5261 // TODO: support for empty() ???
5262 break;
5263 }
5264 op1_info = OP1_INFO();
5265 op1_addr = OP1_REG_ADDR();
5266 if (orig_op1_type != IS_UNKNOWN
5267 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5268 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5269 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5270 goto jit_failure;
5271 }
5272 if (opline->op1_type == IS_CV
5273 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5274 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5275 }
5276 } else {
5277 CHECK_OP1_TRACE_TYPE();
5278 }
5279 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5280 zend_bool exit_if_true = 0;
5281 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5282 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5283
5284 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5285 if (!exit_addr) {
5286 goto jit_failure;
5287 }
5288 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5289 } else {
5290 smart_branch_opcode = 0;
5291 exit_addr = NULL;
5292 }
5293 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
5294 op1_info, op1_addr,
5295 smart_branch_opcode, -1, -1, exit_addr)) {
5296 goto jit_failure;
5297 }
5298 goto done;
5299 case ZEND_IN_ARRAY:
5300 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5301 break;
5302 }
5303 op1_info = OP1_INFO();
5304 op1_addr = OP1_REG_ADDR();
5305 CHECK_OP1_TRACE_TYPE();
5306 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5307 break;
5308 }
5309 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5310 zend_bool exit_if_true = 0;
5311 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5312 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5313
5314 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5315 if (!exit_addr) {
5316 goto jit_failure;
5317 }
5318 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5319 } else {
5320 smart_branch_opcode = 0;
5321 exit_addr = NULL;
5322 }
5323 if (!zend_jit_in_array(&dasm_state, opline,
5324 op1_info, op1_addr,
5325 smart_branch_opcode, -1, -1, exit_addr)) {
5326 goto jit_failure;
5327 }
5328 goto done;
5329 case ZEND_FETCH_DIM_FUNC_ARG:
5330 if (!JIT_G(current_frame)
5331 || !JIT_G(current_frame)->call
5332 || !JIT_G(current_frame)->call->func
5333 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5334 break;
5335 }
5336 /* break missing intentionally */
5337 case ZEND_FETCH_DIM_R:
5338 case ZEND_FETCH_DIM_IS:
5339 case ZEND_FETCH_LIST_R:
5340 op1_info = OP1_INFO();
5341 op1_addr = OP1_REG_ADDR();
5342 if (orig_op1_type != IS_UNKNOWN
5343 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5344 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5345 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5346 goto jit_failure;
5347 }
5348 if (opline->op1_type == IS_CV
5349 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5350 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5351 }
5352 } else {
5353 CHECK_OP1_TRACE_TYPE();
5354 }
5355 op2_info = OP2_INFO();
5356 CHECK_OP2_TRACE_TYPE();
5357 res_info = RES_INFO();
5358 avoid_refcounting =
5359 ssa_op->op1_use >= 0 &&
5360 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5361 if (op1_info & MAY_BE_PACKED_GUARD) {
5362 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5363 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5364 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5365 && (op1_info & MAY_BE_ARRAY_PACKED)
5366 && (op1_info & MAY_BE_ARRAY_HASH)
5367 && orig_op1_type != IS_UNKNOWN) {
5368 op1_info |= MAY_BE_PACKED_GUARD;
5369 if (orig_op1_type & IS_TRACE_PACKED) {
5370 op1_info &= ~MAY_BE_ARRAY_HASH;
5371 } else {
5372 op1_info &= ~MAY_BE_ARRAY_PACKED;
5373 }
5374 }
5375 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
5376 op1_info, op1_addr, avoid_refcounting,
5377 op2_info, res_info, RES_REG_ADDR(),
5378 (
5379 (op1_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
5380 (op2_info & (MAY_BE_ANY - (MAY_BE_LONG|MAY_BE_STRING))) != 0 ||
5381 ((op1_info & MAY_BE_UNDEF) != 0 &&
5382 opline->opcode != ZEND_FETCH_DIM_IS) ||
5383 (opline->opcode != ZEND_FETCH_LIST_R &&
5384 (opline->op1_type & (IS_TMP_VAR|IS_VAR)) != 0 &&
5385 (op1_info & MAY_BE_RC1) &&
5386 (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)) != 0) ||
5387 (op2_info & MAY_BE_UNDEF) != 0 ||
5388 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) != 0 &&
5389 (op2_info & MAY_BE_RC1) &&
5390 (op2_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)) != 0)))) {
5391 goto jit_failure;
5392 }
5393 goto done;
5394 case ZEND_FETCH_DIM_W:
5395 case ZEND_FETCH_DIM_RW:
5396 // case ZEND_FETCH_DIM_UNSET:
5397 case ZEND_FETCH_LIST_W:
5398 if (opline->op1_type != IS_CV
5399 && (orig_op1_type == IS_UNKNOWN
5400 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5401 break;
5402 }
5403 op1_info = OP1_INFO();
5404 op1_addr = OP1_REG_ADDR();
5405 if (opline->op1_type == IS_VAR) {
5406 if (orig_op1_type != IS_UNKNOWN
5407 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5408 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5409 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5410 goto jit_failure;
5411 }
5412 } else {
5413 break;
5414 }
5415 }
5416 if (orig_op1_type != IS_UNKNOWN
5417 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5418 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5419 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5420 goto jit_failure;
5421 }
5422 if (opline->op1_type == IS_CV
5423 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5424 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5425 }
5426 } else {
5427 CHECK_OP1_TRACE_TYPE();
5428 }
5429 op2_info = OP2_INFO();
5430 CHECK_OP2_TRACE_TYPE();
5431 op1_def_info = OP1_DEF_INFO();
5432 if (!zend_jit_fetch_dim(&dasm_state, opline,
5433 op1_info, op1_addr, op2_info, RES_REG_ADDR(),
5434 (opline->opcode == ZEND_FETCH_DIM_RW
5435 || opline->op2_type == IS_UNUSED
5436 || (op1_info & (MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5437 || (op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))
5438 || (opline->op1_type == IS_VAR
5439 && (op1_info & MAY_BE_UNDEF)
5440 && !ssa->var_info[ssa_op->op1_use].indirect_reference)))) {
5441 goto jit_failure;
5442 }
5443 if (ssa_op->result_def > 0
5444 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5445 && !(op1_info & (MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5446 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5447 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5448 }
5449 goto done;
5450 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5451 if ((opline->extended_value & ZEND_ISEMPTY)) {
5452 // TODO: support for empty() ???
5453 break;
5454 }
5455 op1_info = OP1_INFO();
5456 op1_addr = OP1_REG_ADDR();
5457 if (orig_op1_type != IS_UNKNOWN
5458 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5459 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5460 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5461 goto jit_failure;
5462 }
5463 if (opline->op1_type == IS_CV
5464 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5465 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5466 }
5467 } else {
5468 CHECK_OP1_TRACE_TYPE();
5469 }
5470 op2_info = OP2_INFO();
5471 CHECK_OP2_TRACE_TYPE();
5472 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5473 zend_bool exit_if_true = 0;
5474 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5475 uint32_t exit_point;
5476
5477 if (ra) {
5478 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5479 }
5480 if (ssa_op->op1_use >= 0
5481 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5482 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5483 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5484 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5485
5486 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5487 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5488 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5489 } else {
5490 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5491 }
5492 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5493 if (!exit_addr) {
5494 goto jit_failure;
5495 }
5496 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5497 } else {
5498 smart_branch_opcode = 0;
5499 exit_addr = NULL;
5500 }
5501 avoid_refcounting =
5502 ssa_op->op1_use >= 0 &&
5503 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5504 if (op1_info & MAY_BE_PACKED_GUARD) {
5505 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5506 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5507 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5508 && (op1_info & MAY_BE_ARRAY_PACKED)
5509 && (op1_info & MAY_BE_ARRAY_HASH)
5510 && orig_op1_type != IS_UNKNOWN) {
5511 op1_info |= MAY_BE_PACKED_GUARD;
5512 if (orig_op1_type & IS_TRACE_PACKED) {
5513 op1_info &= ~MAY_BE_ARRAY_HASH;
5514 } else {
5515 op1_info &= ~MAY_BE_ARRAY_PACKED;
5516 }
5517 }
5518 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
5519 op1_info, op1_addr, avoid_refcounting,
5520 op2_info,
5521 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5522 smart_branch_opcode, -1, -1,
5523 exit_addr)) {
5524 goto jit_failure;
5525 }
5526 goto done;
5527 case ZEND_FETCH_OBJ_FUNC_ARG:
5528 if (!JIT_G(current_frame)
5529 || !JIT_G(current_frame)->call
5530 || !JIT_G(current_frame)->call->func
5531 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5532 break;
5533 }
5534 /* break missing intentionally */
5535 case ZEND_FETCH_OBJ_R:
5536 case ZEND_FETCH_OBJ_IS:
5537 case ZEND_FETCH_OBJ_W:
5538 delayed_fetch_this = 0;
5539 avoid_refcounting = 0;
5540 if (opline->op2_type != IS_CONST
5541 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5542 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5543 break;
5544 }
5545 ce = NULL;
5546 ce_is_instanceof = 0;
5547 op1_indirect = 0;
5548 if (opline->op1_type == IS_UNUSED) {
5549 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
5550 ce = op_array->scope;
5551 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
5552 op1_addr = 0;
5553 } else {
5554 op1_info = OP1_INFO();
5555 if (!(op1_info & MAY_BE_OBJECT)) {
5556 break;
5557 }
5558 op1_addr = OP1_REG_ADDR();
5559 if (opline->op1_type == IS_VAR
5560 && opline->opcode == ZEND_FETCH_OBJ_W) {
5561 if (orig_op1_type != IS_UNKNOWN
5562 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5563 op1_indirect = 1;
5564 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5565 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5566 goto jit_failure;
5567 }
5568 }
5569 }
5570 if (orig_op1_type != IS_UNKNOWN
5571 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5572 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5573 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5574 goto jit_failure;
5575 }
5576 if (opline->op1_type == IS_CV
5577 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5578 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
5579 }
5580 } else {
5581 CHECK_OP1_TRACE_TYPE();
5582 }
5583 if (!(op1_info & MAY_BE_OBJECT)) {
5584 break;
5585 }
5586 if (ssa->var_info && ssa->ops) {
5587 if (ssa_op->op1_use >= 0) {
5588 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5589 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5590 ce = op1_ssa->ce;
5591 ce_is_instanceof = op1_ssa->is_instanceof;
5592 }
5593 }
5594 }
5595 if (ssa_op->op1_use >= 0) {
5596 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
5597 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5598 }
5599 }
5600 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
5601 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
5602 delayed_fetch_this, avoid_refcounting, op1_ce,
5603 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
5604 goto jit_failure;
5605 }
5606 goto done;
5607 case ZEND_BIND_GLOBAL:
5608 orig_opline = opline;
5609 orig_ssa_op = ssa_op;
5610 while (1) {
5611 if (!ssa->ops || !ssa->var_info) {
5612 op1_info = MAY_BE_ANY|MAY_BE_REF;
5613 } else {
5614 op1_info = OP1_INFO();
5615 }
5616 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5617 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5618 }
5619 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
5620 goto jit_failure;
5621 }
5622 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
5623 opline++;
5624 ssa_op++;
5625 } else {
5626 break;
5627 }
5628 }
5629 opline = orig_opline;
5630 ssa_op = orig_ssa_op;
5631 goto done;
5632 case ZEND_RECV:
5633 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
5634 goto jit_failure;
5635 }
5636 goto done;
5637 case ZEND_RECV_INIT:
5638 orig_opline = opline;
5639 orig_ssa_op = ssa_op;
5640 while (1) {
5641 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
5642 (opline + 1)->opcode != ZEND_RECV_INIT,
5643 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5644 goto jit_failure;
5645 }
5646 if ((opline+1)->opcode == ZEND_RECV_INIT) {
5647 opline++;
5648 ssa_op++;
5649 } else {
5650 break;
5651 }
5652 }
5653 opline = orig_opline;
5654 ssa_op = orig_ssa_op;
5655 goto done;
5656 case ZEND_FREE:
5657 case ZEND_FE_FREE:
5658 op1_info = OP1_INFO();
5659 if (!zend_jit_free(&dasm_state, opline, op1_info,
5660 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5661 goto jit_failure;
5662 }
5663 goto done;
5664 case ZEND_ECHO:
5665 op1_info = OP1_INFO();
5666 CHECK_OP1_TRACE_TYPE();
5667 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5668 break;
5669 }
5670 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
5671 goto jit_failure;
5672 }
5673 goto done;
5674 case ZEND_STRLEN:
5675 op1_info = OP1_INFO();
5676 op1_addr = OP1_REG_ADDR();
5677 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
5678 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5679 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5680 goto jit_failure;
5681 }
5682 if (opline->op1_type == IS_CV
5683 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5684 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5685 }
5686 } else {
5687 CHECK_OP1_TRACE_TYPE();
5688 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5689 break;
5690 }
5691 }
5692 if (!zend_jit_strlen(&dasm_state, opline, op1_info, op1_addr)) {
5693 goto jit_failure;
5694 }
5695 goto done;
5696 case ZEND_FETCH_THIS:
5697 delayed_fetch_this = 0;
5698 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
5699 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
5700 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
5701 delayed_fetch_this = 1;
5702 }
5703 }
5704 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) {
5705 goto jit_failure;
5706 }
5707 goto done;
5708 case ZEND_SWITCH_LONG:
5709 case ZEND_SWITCH_STRING:
5710 case ZEND_MATCH:
5711 if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
5712 goto jit_failure;
5713 }
5714 goto done;
5715 case ZEND_VERIFY_RETURN_TYPE:
5716 if (opline->op1_type == IS_UNUSED) {
5717 /* Always throws */
5718 break;
5719 }
5720 if (opline->op1_type == IS_CONST) {
5721 /* TODO Different instruction format, has return value */
5722 break;
5723 }
5724 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
5725 /* Not worth bothering with */
5726 break;
5727 }
5728 op1_info = OP1_INFO();
5729 CHECK_OP1_TRACE_TYPE();
5730 if (op1_info & MAY_BE_REF) {
5731 /* TODO May need reference unwrapping. */
5732 break;
5733 }
5734 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, op1_info)) {
5735 goto jit_failure;
5736 }
5737 goto done;
5738 case ZEND_FE_RESET_R:
5739 op1_info = OP1_INFO();
5740 CHECK_OP1_TRACE_TYPE();
5741 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
5742 break;
5743 }
5744 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
5745 goto jit_failure;
5746 }
5747 goto done;
5748 case ZEND_FE_FETCH_R:
5749 op1_info = OP1_INFO();
5750 CHECK_OP1_TRACE_TYPE();
5751 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
5752 break;
5753 }
5754 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5755 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
5756 uint32_t exit_point;
5757
5758 if ((p+1)->opline == exit_opline) {
5759 /* taken branch (exit from loop) */
5760 exit_opline = opline;
5761 smart_branch_opcode = ZEND_NOP;
5762 } else if ((p+1)->opline == opline + 1) {
5763 /* not taken branch (loop) */
5764 smart_branch_opcode = ZEND_JMP;
5765 } else {
5766 ZEND_UNREACHABLE();
5767 }
5768 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5769 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5770 if (!exit_addr) {
5771 goto jit_failure;
5772 }
5773 } else {
5774 ZEND_UNREACHABLE();
5775 }
5776 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
5777 -1, smart_branch_opcode, exit_addr)) {
5778 goto jit_failure;
5779 }
5780 goto done;
5781 case ZEND_FETCH_CONSTANT:
5782 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op)) {
5783 goto jit_failure;
5784 }
5785 goto done;
5786 case ZEND_INIT_METHOD_CALL:
5787 if (opline->op2_type != IS_CONST
5788 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
5789 goto generic_dynamic_call;
5790 }
5791 delayed_fetch_this = 0;
5792 ce = NULL;
5793 ce_is_instanceof = 0;
5794 if (opline->op1_type == IS_UNUSED) {
5795 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
5796 ce = op_array->scope;
5797 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
5798 op1_addr = 0;
5799 } else {
5800 op1_info = OP1_INFO();
5801 op1_addr = OP1_REG_ADDR();
5802 if (polymorphic_side_trace) {
5803 op1_info = MAY_BE_OBJECT;
5804 op1_addr = 0;
5805 } else if (orig_op1_type != IS_UNKNOWN
5806 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5807 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5808 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5809 goto jit_failure;
5810 }
5811 if (opline->op1_type == IS_CV
5812 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5813 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5814 }
5815 } else {
5816 CHECK_OP1_TRACE_TYPE();
5817 }
5818 if (ssa->var_info && ssa->ops) {
5819 if (ssa_op->op1_use >= 0) {
5820 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5821 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5822 ce = op1_ssa->ce;
5823 ce_is_instanceof = op1_ssa->is_instanceof;
5824 }
5825 }
5826 }
5827 if (ssa_op->op1_use >= 0) {
5828 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
5829 }
5830 }
5831 frame_flags = TRACE_FRAME_MASK_NESTED;
5832 if (!zend_jit_init_method_call(&dasm_state, opline,
5833 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
5834 op_array, ssa, ssa_op, frame->call_level,
5835 op1_info, op1_addr, ce, ce_is_instanceof, delayed_fetch_this, op1_ce,
5836 p + 1, used_stack < 0, polymorphic_side_trace)) {
5837 goto jit_failure;
5838 }
5839 goto done;
5840 case ZEND_INIT_DYNAMIC_CALL:
5841 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
5842 goto generic_dynamic_call;
5843 }
5844 op2_info = OP2_INFO();
5845 CHECK_OP2_TRACE_TYPE();
5846 frame_flags = TRACE_FRAME_MASK_NESTED;
5847 if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, used_stack < 0)) {
5848 goto jit_failure;
5849 }
5850 goto done;
5851 case ZEND_INIT_STATIC_METHOD_CALL:
5852 generic_dynamic_call:
5853 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5854 goto jit_failure;
5855 }
5856 if ((opline->opcode != ZEND_INIT_STATIC_METHOD_CALL
5857 || opline->op1_type != IS_CONST
5858 || opline->op2_type != IS_CONST)
5859 && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
5860 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
5861 goto jit_failure;
5862 }
5863 }
5864 goto done;
5865 case ZEND_INIT_USER_CALL:
5866 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5867 goto jit_failure;
5868 }
5869 if (opline->op2_type != IS_CONST
5870 && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
5871 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
5872 goto jit_failure;
5873 }
5874 }
5875 goto done;
5876 case ZEND_NEW:
5877 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5878 goto jit_failure;
5879 }
5880 if (opline->op1_type != IS_CONST
5881 && (p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
5882 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
5883 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
5884 goto jit_failure;
5885 }
5886 }
5887 goto done;
5888 case ZEND_SEND_ARRAY:
5889 case ZEND_SEND_UNPACK:
5890 if (JIT_G(current_frame)
5891 && JIT_G(current_frame)->call) {
5892 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5893 }
5894 break;
5895 default:
5896 break;
5897 }
5898 }
5899
5900 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
5901 op1_info = OP1_INFO();
5902 op2_info = OP2_INFO();
5903 if (op1_info & MAY_BE_GUARD) {
5904 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
5905 }
5906 if (op2_info & MAY_BE_GUARD) {
5907 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
5908 }
5909 if (!zend_jit_trace_handler(&dasm_state, op_array, opline,
5910 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
5911 goto jit_failure;
5912 }
5913 }
5914
5915 done:
5916 polymorphic_side_trace = 0;
5917 switch (opline->opcode) {
5918 case ZEND_DO_FCALL:
5919 case ZEND_DO_ICALL:
5920 case ZEND_DO_UCALL:
5921 case ZEND_DO_FCALL_BY_NAME:
5922 frame->call_level--;
5923 }
5924
5925 if (ra) {
5926 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5927 }
5928
5929 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
5930 && STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var)) > ZREG_NUM) {
5931 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5932 }
5933
5934 if (opline->opcode == ZEND_ROPE_INIT) {
5935 /* clear stack slots used by rope */
5936 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
5937 uint32_t count =
5938 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
5939
5940 do {
5941 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
5942 var++;
5943 count--;
5944 } while (count);
5945 }
5946
5947 if (ssa_op) {
5948 zend_ssa_range tmp;
5949
5950 /* Keep information about known types on abstract stack */
5951 if (ssa_op->result_def >= 0) {
5952 zend_uchar type = IS_UNKNOWN;
5953
5954 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
5955 || send_result) {
5956 /* we didn't set result variable */
5957 type = IS_UNKNOWN;
5958 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
5959 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
5960 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
5961 } else if (opline->opcode == ZEND_QM_ASSIGN) {
5962 if (opline->op1_type != IS_CONST) {
5963 /* copy */
5964 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
5965 }
5966 } else if (opline->opcode == ZEND_ASSIGN) {
5967 if (opline->op2_type != IS_CONST
5968 && ssa_op->op1_use >= 0
5969 /* assignment to typed reference may cause conversion */
5970 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
5971 /* copy */
5972 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
5973 }
5974 } else if (opline->opcode == ZEND_POST_INC
5975 || opline->opcode == ZEND_POST_DEC) {
5976 /* copy */
5977 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
5978 }
5979 if (opline->opcode == ZEND_JMP_SET
5980 || opline->opcode == ZEND_COALESCE
5981 || opline->opcode == ZEND_JMP_NULL) {
5982 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
5983 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5984 } else if ((p+1)->opline != (opline + 1)) {
5985 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
5986 }
5987 } else {
5988 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
5989 (type == IS_UNKNOWN || !ra || !ra[ssa_op->result_def]));
5990 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
5991 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
5992 }
5993 if (type != IS_UNKNOWN) {
5994 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
5995 if (opline->opcode == ZEND_FETCH_THIS
5996 && delayed_fetch_this) {
5997 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_THIS);
5998 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
5999 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_TRY_ADDREF);
6000 } else if (ra && ra[ssa_op->result_def]) {
6001 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6002 ra[ssa_op->result_def]->flags & ZREG_STORE);
6003 }
6004 }
6005 }
6006
6007 if (type == IS_LONG
6008 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->result_def, &tmp)) {
6009 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6010 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6011 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6012 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6013 ssa->var_info[ssa_op->result_def].has_range = 1;
6014 }
6015 }
6016 if (ssa_op->op1_def >= 0
6017 && (opline->opcode != ZEND_QM_ASSIGN
6018 || opline->result_type != IS_CV
6019 || opline->result.var != opline->op1.var)) {
6020 zend_uchar type = IS_UNKNOWN;
6021
6022 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6023 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6024 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6025 } else if (opline->opcode == ZEND_ASSIGN) {
6026 if (!(OP1_INFO() & MAY_BE_REF)
6027 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6028 if (opline->op2_type != IS_CONST) {
6029 /* copy */
6030 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6031 }
6032 }
6033 } else if (opline->opcode == ZEND_SEND_VAR
6034 || opline->opcode == ZEND_CAST
6035 || opline->opcode == ZEND_QM_ASSIGN
6036 || opline->opcode == ZEND_JMP_SET
6037 || opline->opcode == ZEND_COALESCE
6038 || opline->opcode == ZEND_JMP_NULL
6039 || opline->opcode == ZEND_FE_RESET_R) {
6040 /* keep old value */
6041 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6042 }
6043 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6044 (type == IS_UNKNOWN || !ra ||
6045 (!ra[ssa_op->op1_def] &&
6046 (opline->opcode == ZEND_ASSIGN || !ssa->vars[ssa_op->op1_def].no_val))));
6047 if (type != IS_UNKNOWN) {
6048 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6049 if (ra && ra[ssa_op->op1_def]) {
6050 uint8_t flags = ra[ssa_op->op1_def]->flags & ZREG_STORE;
6051
6052 if (ssa_op->op1_use >= 0) {
6053 if (opline->opcode == ZEND_SEND_VAR
6054 || opline->opcode == ZEND_CAST
6055 || opline->opcode == ZEND_QM_ASSIGN
6056 || opline->opcode == ZEND_JMP_SET
6057 || opline->opcode == ZEND_COALESCE
6058 || opline->opcode == ZEND_JMP_NULL
6059 || opline->opcode == ZEND_FE_RESET_R) {
6060 if (!ra[ssa_op->op1_use]) {
6061 flags |= ZREG_LOAD;
6062 }
6063 }
6064 }
6065 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, flags);
6066 }
6067 }
6068 if (type == IS_LONG
6069 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op1_def, &tmp)) {
6070 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6071 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6072 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6073 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6074 ssa->var_info[ssa_op->op1_def].has_range = 1;
6075 }
6076 }
6077 if (ssa_op->op2_def >= 0
6078 && (opline->opcode != ZEND_ASSIGN
6079 || opline->op1_type != IS_CV
6080 || opline->op1.var != opline->op2.var)) {
6081 zend_uchar type = IS_UNKNOWN;
6082
6083 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6084 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6085 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6086 } else if (opline->opcode == ZEND_ASSIGN) {
6087 /* keep old value */
6088 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6089 }
6090 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6091 (type == IS_UNKNOWN || !ra ||
6092 (!ra[ssa_op->op2_def] && !ssa->vars[ssa_op->op2_def].no_val)));
6093 if (type != IS_UNKNOWN) {
6094 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6095 if (ra && ra[ssa_op->op2_def]) {
6096 uint8_t flags = ra[ssa_op->op2_def]->flags & ZREG_STORE;
6097
6098 if (ssa_op->op2_use >= 0) {
6099 if (opline->opcode == ZEND_ASSIGN) {
6100 if (!ra[ssa_op->op2_use]
6101 || ra[ssa_op->op2_use]->reg != ra[ssa_op->op2_def]->reg) {
6102 flags |= ZREG_LOAD;
6103 }
6104 }
6105 }
6106 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def]->reg, flags);
6107 }
6108 }
6109 if (type == IS_LONG
6110 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op2_def, &tmp)) {
6111 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6112 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6113 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6114 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6115 ssa->var_info[ssa_op->op2_def].has_range = 1;
6116 }
6117 }
6118
6119 switch (opline->opcode) {
6120 case ZEND_ASSIGN_DIM:
6121 case ZEND_ASSIGN_OBJ:
6122 case ZEND_ASSIGN_STATIC_PROP:
6123 case ZEND_ASSIGN_DIM_OP:
6124 case ZEND_ASSIGN_OBJ_OP:
6125 case ZEND_ASSIGN_STATIC_PROP_OP:
6126 case ZEND_ASSIGN_OBJ_REF:
6127 case ZEND_ASSIGN_STATIC_PROP_REF:
6128 /* OP_DATA */
6129 ssa_op++;
6130 opline++;
6131 if (ssa_op->op1_def >= 0) {
6132 zend_uchar type = IS_UNKNOWN;
6133
6134 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6135 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6136 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6137 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6138 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6139 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6140 /* keep old value */
6141 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6142 }
6143 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6144 (type == IS_UNKNOWN || !ra || !ra[ssa_op->op1_def]));
6145 if (type != IS_UNKNOWN) {
6146 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6147 if (ra && ra[ssa_op->op1_def]) {
6148 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6149 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6150 }
6151 }
6152 if (type == IS_LONG
6153 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op1_def, &tmp)) {
6154 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6155 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6156 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6157 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6158 ssa->var_info[ssa_op->op1_def].has_range = 1;
6159 }
6160 }
6161 ssa_op++;
6162 break;
6163 case ZEND_RECV_INIT:
6164 ssa_op++;
6165 opline++;
6166 while (opline->opcode == ZEND_RECV_INIT) {
6167 if (ssa_op->result_def >= 0) {
6168 zend_uchar type = IS_UNKNOWN;
6169
6170 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6171 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6172 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6173 }
6174 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6175 (!ra || !ra[ssa_op->result_def]));
6176 if (ra && ra[ssa_op->result_def]) {
6177 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6178 ra[ssa_op->result_def]->flags & ZREG_STORE);
6179 }
6180 }
6181 ssa_op++;
6182 opline++;
6183 }
6184 break;
6185 case ZEND_BIND_GLOBAL:
6186 ssa_op++;
6187 opline++;
6188 while (opline->opcode == ZEND_BIND_GLOBAL) {
6189 if (ssa_op->op1_def >= 0) {
6190 zend_uchar type = IS_UNKNOWN;
6191
6192 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6193 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6194 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6195 }
6196 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6197 (!ra || !ra[ssa_op->op1_def]));
6198 if (ra && ra[ssa_op->op1_def]) {
6199 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6200 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6201 }
6202 }
6203 ssa_op++;
6204 opline++;
6205 }
6206 break;
6207 default:
6208 ssa_op += zend_jit_trace_op_len(opline);
6209 break;
6210 }
6211
6212 if (send_result) {
6213 ssa_op++;
6214 send_result = 0;
6215 }
6216 }
6217 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6218 call = frame->call;
6219 assert(call && &call->func->op_array == p->op_array);
6220
6221 if (opline->opcode == ZEND_DO_UCALL
6222 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6223 || opline->opcode == ZEND_DO_FCALL) {
6224
6225 frame->call_opline = opline;
6226
6227 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6228 if (opline > op_array->opcodes) {
6229 const zend_op *prev_opline = opline - 1;
6230
6231 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6232 prev_opline--;
6233 }
6234 JIT_G(current_frame) = call;
6235 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6236 || prev_opline->opcode == ZEND_SEND_UNPACK
6237 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6238 && p->op_array->num_args
6239 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6240 && ((p+1)->op == ZEND_JIT_TRACE_VM
6241 || (p+1)->op == ZEND_JIT_TRACE_END)
6242 && (TRACE_FRAME_NUM_ARGS(call) < 0
6243 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6244 && !zend_jit_trace_opline_guard(&dasm_state, (p+1)->opline)) {
6245 goto jit_failure;
6246 }
6247 JIT_G(current_frame) = frame;
6248 }
6249 }
6250
6251 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6252 p++;
6253 break;
6254 }
6255 op_array = (zend_op_array*)p->op_array;
6256 jit_extension =
6257 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6258 op_array_ssa = &jit_extension->func_info.ssa;
6259 frame->call = call->prev;
6260 call->prev = frame;
6261 if (p->info & ZEND_JIT_TRACE_RETRUN_VALUE_USED) {
6262 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6263 } else {
6264 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6265 }
6266 JIT_G(current_frame) = frame = call;
6267 stack = frame->stack;
6268 if (ra) {
6269 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6270
6271 for (i = 0; i < op_array->last_var; i++,j++) {
6272 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6273 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6274 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6275 goto jit_failure;
6276 }
6277 }
6278 }
6279 }
6280 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6281 op_array = (zend_op_array*)p->op_array;
6282 jit_extension =
6283 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6284 op_array_ssa = &jit_extension->func_info.ssa;
6285 top = frame;
6286 if (frame->prev) {
6287 frame = frame->prev;
6288 stack = frame->stack;
6289 ZEND_ASSERT(&frame->func->op_array == op_array);
6290 } else {
6291 frame = zend_jit_trace_ret_frame(frame, op_array);
6292 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6293 stack = frame->stack;
6294 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6295 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6296
6297 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6298 /* Initialize abstract stack using SSA */
6299 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6300 && has_concrete_type(ssa->var_info[j].type)) {
6301 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6302 } else {
6303 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6304 }
6305 }
6306 if (ra) {
6307 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6308 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6309 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6310 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6311 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6312 goto jit_failure;
6313 }
6314 }
6315 }
6316 }
6317 } else {
6318 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6319 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6320 }
6321 }
6322 opline = NULL;
6323 }
6324 JIT_G(current_frame) = frame;
6325 if (res_type != IS_UNKNOWN
6326 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6327 const zend_op *opline = (p+1)->opline - 1;
6328 if (opline->result_type != IS_UNUSED) {
6329 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6330 }
6331 }
6332 res_type = IS_UNKNOWN;
6333 } else if (p->op == ZEND_JIT_TRACE_END) {
6334 break;
6335 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6336 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6337 int num_args = -1;
6338
6339 if (init_opline
6340 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6341 num_args = init_opline->extended_value;
6342 }
6343
6344 call = top;
6345 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6346 call->prev = frame->call;
6347 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6348 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6349 }
6350 if (init_opline
6351 && init_opline->opcode != ZEND_NEW
6352 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6353 || init_opline->op1_type == IS_UNDEF)
6354 && (init_opline->opcode != ZEND_INIT_USER_CALL
6355 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6356 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6357 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6358 ) {
6359 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6360 }
6361 frame->call = call;
6362 top = zend_jit_trace_call_frame(top, p->op_array);
6363 if (p->func) {
6364 if (p->func->type == ZEND_USER_FUNCTION) {
6365 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6366 zend_jit_op_array_trace_extension *jit_extension =
6367 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6368
6369 i = 0;
6370 while (i < p->op_array->num_args) {
6371 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6372 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6373 i++;
6374 }
6375 while (i < p->op_array->last_var) {
6376 if (jit_extension
6377 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6378 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6379 } else {
6380 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6381 }
6382 i++;
6383 }
6384 while (i < p->op_array->last_var + p->op_array->T) {
6385 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6386 i++;
6387 }
6388 } else {
6389 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
6390 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6391 }
6392 }
6393 }
6394 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6395 int skip_guard = 0;
6396
6397 if (init_opline) {
6398 zend_call_info *call_info = jit_extension->func_info.callee_info;
6399
6400 while (call_info) {
6401 if (call_info->caller_init_opline == init_opline) {
6402 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
6403 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
6404 && init_opline->op1_type != IS_CONST) {
6405 break;
6406 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
6407 break;
6408 }
6409 }
6410 skip_guard = 1;
6411 break;
6412 }
6413 call_info = call_info->next_callee;
6414 }
6415 if (!skip_guard
6416 && !zend_jit_may_be_polymorphic_call(init_opline)) {
6417 // TODO: recompilation may change target ???
6418 skip_guard = 1;
6419 }
6420 }
6421
6422 if (!skip_guard) {
6423 if (!opline) {
6424 zend_jit_trace_rec *q = p + 1;
6425 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
6426 q++;
6427 }
6428 opline = q->opline;
6429 ZEND_ASSERT(opline != NULL);
6430 }
6431 if (!zend_jit_init_fcall_guard(&dasm_state,
6432 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
6433 goto jit_failure;
6434 }
6435 }
6436 }
6437 }
6438 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6439 frame->call_level++;
6440 }
6441 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
6442 call = frame->call;
6443 if (call) {
6444 top = call;
6445 frame->call = call->prev;
6446 }
6447 } else {
6448 ZEND_UNREACHABLE();
6449 }
6450 }
6451
6452 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
6453
6454 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
6455
6456 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6457 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6458 }
6459
6460 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
6461 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
6462 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6463 if (ra) {
6464 zend_ssa_phi *phi = ssa->blocks[1].phis;
6465
6466 while (phi) {
6467 if (ra[phi->ssa_var]
6468 && ra[phi->sources[1]]
6469 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
6470 && (ra[phi->ssa_var]->flags & (ZREG_LOAD|ZREG_STORE)) == 0
6471 && (ra[phi->sources[1]]->flags & (ZREG_LOAD|ZREG_STORE)) == 0) {
6472 /* Store actual type to memory to avoid deoptimization mistakes */
6473 /* TODO: Alternatively, we may try to update alredy generated deoptimization info */
6474 zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var));
6475 }
6476 phi = phi->next;
6477 }
6478 }
6479 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6480 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6481 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6482 goto jit_failure;
6483 }
6484 }
6485 t->link = ZEND_JIT_TRACE_NUM;
6486 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6487 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
6488 }
6489 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
6490 const void *timeout_exit_addr = NULL;
6491
6492 t->flags |= ZEND_JIT_TRACE_LOOP;
6493
6494 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6495 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6496 || (ra
6497 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
6498 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
6499
6500 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6501 if (!timeout_exit_addr) {
6502 goto jit_failure;
6503 }
6504 } else {
6505 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6506 }
6507 }
6508
6509 zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
6510 }
6511 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
6512 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
6513 if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL,
6514 stack, op_array->last_var + op_array->T, NULL, NULL, NULL, 0)) {
6515 goto jit_failure;
6516 }
6517 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
6518 const void *timeout_exit_addr = NULL;
6519
6520 t->link = zend_jit_find_trace(p->opline->handler);
6521 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6522 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6523 goto jit_failure;
6524 }
6525 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6526 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6527 }
6528 if (parent_trace
6529 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
6530 && zend_jit_traces[parent_trace].root == t->link) {
6531 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
6532 uint32_t exit_point;
6533
6534 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6535 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6536 }
6537 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
6538 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6539 if (!timeout_exit_addr) {
6540 goto jit_failure;
6541 }
6542 } else {
6543 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6544 }
6545 }
6546 zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
6547 } else {
6548 zend_jit_trace_return(&dasm_state, 0, NULL);
6549 }
6550 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
6551 zend_jit_trace_return(&dasm_state, 0, NULL);
6552 } else {
6553 // TODO: not implemented ???
6554 ZEND_ASSERT(0 && p->stop);
6555 }
6556
6557 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
6558 goto jit_failure;
6559 }
6560
6561 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM);
6562
6563 if (handler) {
6564 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
6565 const zend_op_array *rec_op_array;
6566
6567 rec_op_array = op_array = trace_buffer->op_array;
6568 jit_extension =
6569 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6570 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
6571 for (;;p++) {
6572 if (p->op == ZEND_JIT_TRACE_VM) {
6573 opline = p->opline;
6574 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6575 if (p->op_array == rec_op_array) {
6576 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
6577 }
6578 op_array = p->op_array;
6579 jit_extension =
6580 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6581 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6582 op_array = p->op_array;
6583 jit_extension =
6584 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6585 } else if (p->op == ZEND_JIT_TRACE_END) {
6586 break;
6587 }
6588 }
6589 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
6590 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
6591 if (opline
6592 && (opline->opcode == ZEND_DO_UCALL
6593 || opline->opcode == ZEND_DO_FCALL
6594 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6595 || opline->opcode == ZEND_YIELD
6596 || opline->opcode == ZEND_YIELD_FROM
6597 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
6598 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
6599 }
6600 if (JIT_G(current_frame)
6601 && JIT_G(current_frame)->prev) {
6602 frame = JIT_G(current_frame)->prev;
6603 do {
6604 if (frame->call_opline) {
6605 op_array = &frame->func->op_array;
6606 jit_extension =
6607 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6608 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
6609 }
6610 frame = frame->prev;
6611 } while (frame);
6612 }
6613 }
6614 }
6615
6616 jit_failure:
6617 dasm_free(&dasm_state);
6618
6619 if (name) {
6620 zend_string_release(name);
6621 }
6622
6623 jit_cleanup:
6624 /* Clean up used op_arrays */
6625 while (num_op_arrays > 0) {
6626 op_array = op_arrays[--num_op_arrays];
6627 jit_extension =
6628 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6629
6630 jit_extension->func_info.num = 0;
6631 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
6632 | ZEND_FUNC_JIT_ON_PROF_REQUEST
6633 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
6634 | ZEND_FUNC_JIT_ON_HOT_TRACE;
6635 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
6636 }
6637
6638 zend_arena_release(&CG(arena), checkpoint);
6639
6640 JIT_G(current_frame) = NULL;
6641 JIT_G(current_trace) = NULL;
6642
6643 return handler;
6644 }
6645
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)6646 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
6647 {
6648 const void *handler = NULL;
6649 dasm_State* dasm_state = NULL;
6650 void *checkpoint;
6651 char name[32];
6652 const zend_op *opline;
6653 uint32_t stack_size;
6654 zend_jit_trace_stack *stack;
6655 zend_bool original_handler = 0;
6656
6657 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
6658 return dasm_labels[zend_lbtrace_escape];
6659 }
6660
6661 checkpoint = zend_arena_checkpoint(CG(arena));;
6662
6663 sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num);
6664
6665 dasm_init(&dasm_state, DASM_MAXSECTION);
6666 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
6667 dasm_setup(&dasm_state, dasm_actions);
6668
6669 zend_jit_align_func(&dasm_state);
6670
6671 /* Deoptimization */
6672 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
6673 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
6674
6675 if (!zend_jit_trace_deoptimization(&dasm_state,
6676 zend_jit_traces[trace_num].exit_info[exit_num].flags,
6677 zend_jit_traces[trace_num].exit_info[exit_num].opline,
6678 stack, stack_size, NULL, NULL, NULL, 0)) {
6679 goto jit_failure;
6680 }
6681
6682 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
6683 if (opline) {
6684 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
6685 /* prevent endless loop */
6686 original_handler = 1;
6687 }
6688 zend_jit_set_ip_ex(&dasm_state, opline, original_handler);
6689 }
6690
6691 zend_jit_trace_return(&dasm_state, original_handler, opline);
6692
6693 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM);
6694
6695 jit_failure:
6696 dasm_free(&dasm_state);
6697 zend_arena_release(&CG(arena), checkpoint);
6698 return handler;
6699 }
6700
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)6701 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
6702 {
6703 zend_jit_trace_stop ret;
6704 const void *handler;
6705 uint8_t orig_trigger;
6706 zend_jit_trace_info *t = NULL;
6707 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
6708
6709 zend_shared_alloc_lock();
6710
6711 /* Checks under lock */
6712 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
6713 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
6714 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
6715 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
6716 } else {
6717 SHM_UNPROTECT();
6718 zend_jit_unprotect();
6719
6720 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
6721
6722 t->id = ZEND_JIT_TRACE_NUM;
6723 t->root = ZEND_JIT_TRACE_NUM;
6724 t->parent = 0;
6725 t->link = 0;
6726 t->exit_count = 0;
6727 t->child_count = 0;
6728 t->stack_map_size = 0;
6729 t->flags = 0;
6730 t->polymorphism = 0;
6731 t->jmp_table_size = 0;
6732 t->opline = trace_buffer[1].opline;
6733 t->exit_info = exit_info;
6734 t->stack_map = NULL;
6735
6736 orig_trigger = JIT_G(trigger);
6737 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
6738
6739 handler = zend_jit_trace(trace_buffer, 0, 0);
6740
6741 JIT_G(trigger) = orig_trigger;
6742
6743 if (handler) {
6744 zend_jit_trace_exit_info *shared_exit_info = NULL;
6745
6746 t->exit_info = NULL;
6747 if (t->exit_count) {
6748 /* reallocate exit_info into shared memory */
6749 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
6750 sizeof(zend_jit_trace_exit_info) * t->exit_count);
6751
6752 if (!shared_exit_info) {
6753 if (t->stack_map) {
6754 efree(t->stack_map);
6755 t->stack_map = NULL;
6756 }
6757 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
6758 goto exit;
6759 }
6760 memcpy(shared_exit_info, exit_info,
6761 sizeof(zend_jit_trace_exit_info) * t->exit_count);
6762 t->exit_info = shared_exit_info;
6763 }
6764
6765 if (t->stack_map_size) {
6766 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
6767 if (!shared_stack_map) {
6768 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
6769 goto exit;
6770 }
6771 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
6772 efree(t->stack_map);
6773 t->stack_map = shared_stack_map;
6774 }
6775
6776 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
6777 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
6778
6779 ((zend_op*)opline)->handler = handler;
6780
6781 ZEND_JIT_TRACE_NUM++;
6782 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
6783
6784 ret = ZEND_JIT_TRACE_STOP_COMPILED;
6785 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
6786 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
6787 if (t->stack_map) {
6788 efree(t->stack_map);
6789 t->stack_map = NULL;
6790 }
6791 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
6792 } else {
6793 if (t->stack_map) {
6794 efree(t->stack_map);
6795 t->stack_map = NULL;
6796 }
6797 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
6798 }
6799
6800 exit:
6801 zend_jit_protect();
6802 SHM_PROTECT();
6803 }
6804
6805 zend_shared_alloc_unlock();
6806
6807 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
6808 && ret == ZEND_JIT_TRACE_STOP_COMPILED
6809 && t->exit_count > 0) {
6810 zend_jit_dump_exit_info(t);
6811 }
6812
6813 return ret;
6814 }
6815
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)6816 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
6817 {
6818 zend_shared_alloc_lock();
6819
6820 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
6821 SHM_UNPROTECT();
6822 zend_jit_unprotect();
6823
6824 ((zend_op*)opline)->handler =
6825 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
6826
6827 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
6828
6829 zend_jit_protect();
6830 SHM_PROTECT();
6831 }
6832
6833 zend_shared_alloc_unlock();
6834 }
6835
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)6836 static zend_bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
6837 {
6838 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
6839 uint8_t *cache_count = JIT_G(bad_root_cache_count);
6840 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
6841 uint32_t cache_slot = JIT_G(bad_root_slot);
6842 uint32_t i;
6843
6844 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
6845 if (cache_opline[i] == opline) {
6846 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
6847 cache_opline[i] = NULL;
6848 return 1;
6849 } else {
6850 #if 0
6851 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
6852 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
6853 random() % ZEND_JIT_TRACE_COUNTER_MAX;
6854 }
6855 #endif
6856 cache_count[i]++;
6857 cache_stop[i] = stop;
6858 return 0;
6859 }
6860 }
6861 }
6862 i = cache_slot;
6863 cache_opline[i] = opline;
6864 cache_count[i] = 1;
6865 cache_stop[i] = stop;
6866 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
6867 JIT_G(bad_root_slot) = cache_slot;
6868 return 0;
6869 }
6870
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)6871 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
6872 {
6873 zend_jit_trace_rec *p = trace_buffer;
6874 const zend_op_array *op_array;
6875 const zend_op *opline;
6876 uint32_t level = 1 + trace_buffer[0].level;
6877 int idx, len, i, v, vars_count, call_level;
6878
6879 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
6880 op_array = p->op_array;
6881 p += ZEND_JIT_TRACE_START_REC_SIZE;
6882 idx = 0;
6883 call_level = 0;
6884
6885 if (tssa && tssa->var_info) {
6886 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
6887 vars_count = op_array->last_var;
6888 } else {
6889 vars_count = op_array->last_var + op_array->T;
6890 }
6891 for (i = 0; i < vars_count; i++) {
6892 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
6893 fprintf(stderr, " %*c;", level, ' ');
6894 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
6895 fprintf(stderr, "\n");
6896 }
6897 }
6898 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
6899 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
6900 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6901 zend_ssa_phi *p = tssa->blocks[1].phis;
6902
6903 fprintf(stderr, "LOOP:\n");
6904
6905 while (p) {
6906 fprintf(stderr, " ;");
6907 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
6908 fprintf(stderr, " = Phi(");
6909 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
6910 fprintf(stderr, ", ");
6911 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
6912 fprintf(stderr, ")\n");
6913 p = p->next;
6914 }
6915 }
6916 }
6917
6918 while (1) {
6919 if (p->op == ZEND_JIT_TRACE_VM) {
6920 uint8_t op1_type, op2_type, op3_type;
6921
6922 opline = p->opline;
6923 fprintf(stderr, "%04d%*c",
6924 (int)(opline - op_array->opcodes),
6925 level, ' ');
6926 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
6927
6928 op1_type = p->op1_type;
6929 op2_type = p->op2_type;
6930 op3_type = p->op3_type;
6931 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
6932 fprintf(stderr, " ;");
6933 if (op1_type != IS_UNKNOWN) {
6934 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
6935 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
6936 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
6937 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6938 p++;
6939 fprintf(stderr, " op1(%sobject of class %s)", ref,
6940 ZSTR_VAL(p->ce->name));
6941 } else {
6942 const char *type = (op1_type == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
6943 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
6944 }
6945 }
6946 if (op2_type != IS_UNKNOWN) {
6947 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
6948 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
6949 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
6950 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
6951 p++;
6952 fprintf(stderr, " op2(%sobject of class %s)", ref,
6953 ZSTR_VAL(p->ce->name));
6954 } else {
6955 const char *type = (op2_type == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
6956 fprintf(stderr, " op2(%s%s)", ref, type);
6957 }
6958 }
6959 if (op3_type != IS_UNKNOWN) {
6960 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
6961 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
6962 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
6963 const char *type = (op3_type == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
6964 fprintf(stderr, " op3(%s%s)", ref, type);
6965 }
6966 }
6967 fprintf(stderr, "\n");
6968 idx++;
6969
6970 len = zend_jit_trace_op_len(opline);
6971 while (len > 1) {
6972 opline++;
6973 fprintf(stderr, "%04d%*c;",
6974 (int)(opline - op_array->opcodes),
6975 level, ' ');
6976 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
6977 idx++;
6978 len--;
6979 fprintf(stderr, "\n");
6980 }
6981 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6982 op_array = p->op_array;
6983 fprintf(stderr, " %*c>enter %s%s%s\n",
6984 level, ' ',
6985 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
6986 op_array->scope ? "::" : "",
6987 op_array->function_name ?
6988 ZSTR_VAL(op_array->function_name) :
6989 ZSTR_VAL(op_array->filename));
6990 level++;
6991 if (tssa && tssa->var_info) {
6992 call_level++;
6993 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6994 vars_count = op_array->last_var;
6995 for (i = 0; i < vars_count; i++, v++) {
6996 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
6997 fprintf(stderr, " %*c;", level, ' ');
6998 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
6999 fprintf(stderr, "\n");
7000 }
7001 }
7002 }
7003 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7004 op_array = p->op_array;
7005 level--;
7006 fprintf(stderr, " %*c<back %s%s%s\n",
7007 level, ' ',
7008 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7009 op_array->scope ? "::" : "",
7010 op_array->function_name ?
7011 ZSTR_VAL(op_array->function_name) :
7012 ZSTR_VAL(op_array->filename));
7013 if (tssa && tssa->var_info) {
7014 if (call_level == 0) {
7015 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7016 vars_count = op_array->last_var + op_array->T;
7017 for (i = 0; i < vars_count; i++, v++) {
7018 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7019 fprintf(stderr, " %*c;", level, ' ');
7020 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7021 fprintf(stderr, "\n");
7022 }
7023 }
7024 } else {
7025 call_level--;
7026 }
7027 }
7028 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7029 if (p->func != (zend_function*)&zend_pass_function) {
7030 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7031 level, ' ',
7032 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7033 (p->func && p->func->common.scope) ? "::" : "",
7034 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7035 } else {
7036 fprintf(stderr, " %*c>skip\n",
7037 level, ' ');
7038 }
7039 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7040 if (p->func != (zend_function*)&zend_pass_function) {
7041 fprintf(stderr, " %*c>call %s%s%s\n",
7042 level, ' ',
7043 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7044 p->func->common.scope ? "::" : "",
7045 ZSTR_VAL(p->func->common.function_name));
7046 } else {
7047 fprintf(stderr, " %*c>skip\n",
7048 level, ' ');
7049 }
7050 } else if (p->op == ZEND_JIT_TRACE_END) {
7051 break;
7052 }
7053 p++;
7054 }
7055 }
7056
zend_jit_dump_exit_info(zend_jit_trace_info * t)7057 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7058 {
7059 int i, j;
7060
7061 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7062 for (i = 0; i < t->exit_count; i++) {
7063 const zend_op_array *op_array = t->exit_info[i].op_array;
7064 uint32_t stack_size = t->exit_info[i].stack_size;
7065 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset;
7066
7067 fprintf(stderr, " exit_%d:", i);
7068 if (t->exit_info[i].opline) {
7069 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7070 } else {
7071 fprintf(stderr, " ----/");
7072 }
7073 if (t->exit_info[i].stack_size) {
7074 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7075 } else {
7076 fprintf(stderr, "----/0");
7077 }
7078 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7079 fprintf(stderr, "/VM");
7080 }
7081 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7082 fprintf(stderr, "/CALL");
7083 }
7084 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7085 fprintf(stderr, "/POLY");
7086 }
7087 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7088 fprintf(stderr, "/FREE_OP1");
7089 }
7090 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7091 fprintf(stderr, "/FREE_OP2");
7092 }
7093 for (j = 0; j < stack_size; j++) {
7094 zend_uchar type = STACK_TYPE(stack, j);
7095 if (type != IS_UNKNOWN) {
7096 fprintf(stderr, " ");
7097 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7098 fprintf(stderr, ":");
7099 if (type == IS_UNDEF) {
7100 fprintf(stderr, "undef");
7101 } else {
7102 fprintf(stderr, "%s", zend_get_type_by_const(type));
7103 }
7104 if (STACK_REG(stack, j) != ZREG_NONE) {
7105 if (STACK_REG(stack, j) < ZREG_NUM) {
7106 fprintf(stderr, "(%s)", zend_reg_name[STACK_REG(stack, j)]);
7107 } else if (STACK_REG(stack, j) == ZREG_THIS) {
7108 fprintf(stderr, "(this)");
7109 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7110 fprintf(stderr, "(zval_try_addref)");
7111 } else {
7112 fprintf(stderr, "(const_%d)", STACK_REG(stack, j) - ZREG_NUM);
7113 }
7114 }
7115 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7116 fprintf(stderr, " ");
7117 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7118 fprintf(stderr, ":unknown(zval_try_addref)");
7119 } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_R0) {
7120 fprintf(stderr, " ");
7121 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7122 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[0]);
7123 }
7124 }
7125 fprintf(stderr, "\n");
7126 }
7127 }
7128
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)7129 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
7130 {
7131 const zend_op *orig_opline;
7132 zend_jit_trace_stop stop;
7133 int ret = 0;
7134 zend_op_array *op_array;
7135 zend_jit_op_array_trace_extension *jit_extension;
7136 size_t offset;
7137 uint32_t trace_num;
7138 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7139
7140 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
7141 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
7142 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
7143
7144 repeat:
7145 trace_num = ZEND_JIT_TRACE_NUM;
7146 orig_opline = opline;
7147 op_array = &EX(func)->op_array;
7148 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7149 offset = jit_extension->offset;
7150
7151 EX(opline) = opline;
7152
7153 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
7154 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7155 return 0;
7156 }
7157
7158 if (JIT_G(tracing)) {
7159 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
7160 return 0;
7161 }
7162
7163 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7164 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7165 trace_num,
7166 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7167 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7168 EX(func)->op_array.scope ? "::" : "",
7169 EX(func)->op_array.function_name ?
7170 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7171 ZSTR_VAL(EX(func)->op_array.filename),
7172 opline->lineno);
7173 }
7174
7175 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7176 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7177 goto abort;
7178 }
7179
7180 JIT_G(tracing) = 1;
7181 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
7182 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
7183 JIT_G(tracing) = 0;
7184
7185 if (stop & ZEND_JIT_TRACE_HALT) {
7186 ret = -1;
7187 }
7188 stop &= ~ZEND_JIT_TRACE_HALT;
7189
7190 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
7191 orig_opline = trace_buffer[1].opline;
7192 op_array = (zend_op_array*)trace_buffer[0].op_array;
7193 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7194 offset = jit_extension->offset;
7195 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7196 const zend_op_array *op_array = trace_buffer[0].op_array;
7197 const zend_op *opline = trace_buffer[1].opline;
7198 zend_jit_op_array_trace_extension *jit_extension =
7199 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7200 size_t offset = jit_extension->offset;
7201
7202 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7203 trace_num,
7204 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7205 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7206 op_array->scope ? "::" : "",
7207 op_array->function_name ?
7208 ZSTR_VAL(op_array->function_name) : "$main",
7209 ZSTR_VAL(op_array->filename),
7210 opline->lineno);
7211 }
7212 }
7213
7214 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
7215 zend_jit_dump_trace(trace_buffer, NULL);
7216 }
7217
7218 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
7219 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
7220 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
7221 uint32_t idx = trace_buffer[1].last;
7222 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
7223 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
7224 trace_num,
7225 link_to);
7226 } else {
7227 fprintf(stderr, "---- TRACE %d stop (%s)\n",
7228 trace_num,
7229 zend_jit_trace_stop_description[stop]);
7230 }
7231 }
7232 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
7233 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
7234 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
7235 fprintf(stderr, "---- TRACE %d %s\n",
7236 trace_num,
7237 zend_jit_trace_stop_description[stop]);
7238 }
7239 } else {
7240 goto abort;
7241 }
7242 } else {
7243 abort:
7244 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
7245 fprintf(stderr, "---- TRACE %d abort (%s)\n",
7246 trace_num,
7247 zend_jit_trace_stop_description[stop]);
7248 }
7249 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
7250 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
7251 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7252 fprintf(stderr, "---- TRACE %d blacklisted\n",
7253 trace_num);
7254 }
7255 zend_jit_blacklist_root_trace(orig_opline, offset);
7256 }
7257 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
7258 execute_data = EG(current_execute_data);
7259 opline = EX(opline);
7260 goto repeat;
7261 }
7262 }
7263
7264 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
7265 fprintf(stderr, "\n");
7266 }
7267
7268 return ret;
7269 }
7270
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)7271 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
7272 {
7273 const void *handler;
7274
7275 zend_shared_alloc_lock();
7276
7277 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
7278 SHM_UNPROTECT();
7279 zend_jit_unprotect();
7280
7281 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
7282
7283 if (handler) {
7284 zend_jit_link_side_trace(
7285 zend_jit_traces[trace_num].code_start,
7286 zend_jit_traces[trace_num].code_size,
7287 zend_jit_traces[trace_num].jmp_table_size,
7288 exit_num,
7289 handler);
7290 }
7291
7292 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
7293
7294 zend_jit_protect();
7295 SHM_PROTECT();
7296 }
7297
7298 zend_shared_alloc_unlock();
7299 }
7300
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)7301 static zend_bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
7302 {
7303 uint8_t *counter = JIT_G(exit_counters) +
7304 zend_jit_traces[trace_num].exit_counters + exit_num;
7305
7306 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
7307 return 1;
7308 }
7309 (*counter)++;
7310 return 0;
7311 }
7312
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)7313 static zend_bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
7314 {
7315 uint8_t *counter = JIT_G(exit_counters) +
7316 zend_jit_traces[trace_num].exit_counters + exit_num;
7317
7318 if (*counter + 1 >= JIT_G(hot_side_exit)) {
7319 return 1;
7320 }
7321 (*counter)++;
7322 return 0;
7323 }
7324
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)7325 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
7326 {
7327 zend_jit_trace_stop ret;
7328 const void *handler;
7329 uint8_t orig_trigger;
7330 zend_jit_trace_info *t;
7331 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7332
7333 zend_shared_alloc_lock();
7334
7335 /* Checks under lock */
7336 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7337 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7338 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7339 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7340 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
7341 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
7342 } else {
7343 SHM_UNPROTECT();
7344 zend_jit_unprotect();
7345
7346 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7347
7348 t->id = ZEND_JIT_TRACE_NUM;
7349 t->root = zend_jit_traces[parent_num].root;
7350 t->parent = parent_num;
7351 t->link = 0;
7352 t->exit_count = 0;
7353 t->child_count = 0;
7354 t->stack_map_size = 0;
7355 t->flags = 0;
7356 t->polymorphism = polymorphism;
7357 t->jmp_table_size = 0;
7358 t->opline = NULL;
7359 t->exit_info = exit_info;
7360 t->stack_map = NULL;
7361
7362 orig_trigger = JIT_G(trigger);
7363 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7364
7365 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
7366
7367 JIT_G(trigger) = orig_trigger;
7368
7369 if (handler) {
7370 zend_jit_trace_exit_info *shared_exit_info = NULL;
7371
7372 t->exit_info = NULL;
7373 if (t->exit_count) {
7374 /* reallocate exit_info into shared memory */
7375 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7376 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7377
7378 if (!shared_exit_info) {
7379 if (t->stack_map) {
7380 efree(t->stack_map);
7381 t->stack_map = NULL;
7382 }
7383 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7384 goto exit;
7385 }
7386 memcpy(shared_exit_info, exit_info,
7387 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7388 t->exit_info = shared_exit_info;
7389 }
7390
7391 if (t->stack_map_size) {
7392 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7393 if (!shared_stack_map) {
7394 efree(t->stack_map);
7395 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7396 goto exit;
7397 }
7398 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7399 efree(t->stack_map);
7400 t->stack_map = shared_stack_map;
7401 }
7402
7403 zend_jit_link_side_trace(
7404 zend_jit_traces[parent_num].code_start,
7405 zend_jit_traces[parent_num].code_size,
7406 zend_jit_traces[parent_num].jmp_table_size,
7407 exit_num,
7408 handler);
7409
7410 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7411 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7412
7413 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
7414 ZEND_JIT_TRACE_NUM++;
7415 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
7416
7417 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7418 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7419 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7420 if (t->stack_map) {
7421 efree(t->stack_map);
7422 t->stack_map = NULL;
7423 }
7424 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7425 } else {
7426 if (t->stack_map) {
7427 efree(t->stack_map);
7428 t->stack_map = NULL;
7429 }
7430 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7431 }
7432
7433 exit:
7434 zend_jit_protect();
7435 SHM_PROTECT();
7436 }
7437
7438 zend_shared_alloc_unlock();
7439
7440 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7441 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7442 && t->exit_count > 0) {
7443 zend_jit_dump_exit_info(t);
7444 }
7445
7446 return ret;
7447 }
7448
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)7449 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
7450 {
7451 zend_jit_trace_stop stop;
7452 int ret = 0;
7453 uint32_t trace_num;
7454 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7455 uint32_t is_megamorphic = 0;
7456 uint32_t polymorphism = 0;
7457
7458 trace_num = ZEND_JIT_TRACE_NUM;
7459
7460 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
7461 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7462 return 0;
7463 }
7464
7465 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7466 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
7467 trace_num, parent_num, exit_num,
7468 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7469 EX(func)->op_array.scope ? "::" : "",
7470 EX(func)->op_array.function_name ?
7471 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7472 ZSTR_VAL(EX(func)->op_array.filename),
7473 EX(opline)->lineno);
7474 }
7475
7476 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7477 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7478 goto abort;
7479 }
7480
7481 if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
7482 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
7483 goto abort;
7484 }
7485
7486 if (JIT_G(max_polymorphic_calls) > 0) {
7487 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
7488 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
7489 && EX(call))) {
7490 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
7491 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
7492 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
7493 } else if (!zend_jit_traces[parent_num].polymorphism) {
7494 polymorphism = 1;
7495 } else if (exit_num == 0) {
7496 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
7497 }
7498 }
7499 }
7500
7501 JIT_G(tracing) = 1;
7502 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
7503 JIT_G(tracing) = 0;
7504
7505 if (stop & ZEND_JIT_TRACE_HALT) {
7506 ret = -1;
7507 }
7508 stop &= ~ZEND_JIT_TRACE_HALT;
7509
7510 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
7511 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7512 const zend_op_array *op_array = trace_buffer[0].op_array;
7513 const zend_op *opline = trace_buffer[1].opline;
7514 zend_jit_op_array_trace_extension *jit_extension =
7515 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7516 size_t offset = jit_extension->offset;
7517
7518 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7519 trace_num,
7520 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7521 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7522 op_array->scope ? "::" : "",
7523 op_array->function_name ?
7524 ZSTR_VAL(op_array->function_name) : "$main",
7525 ZSTR_VAL(op_array->filename),
7526 opline->lineno);
7527 }
7528 }
7529
7530 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
7531 zend_jit_dump_trace(trace_buffer, NULL);
7532 }
7533
7534 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
7535 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
7536 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
7537 uint32_t idx = trace_buffer[1].last;
7538 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
7539 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
7540 trace_num,
7541 link_to);
7542 } else {
7543 fprintf(stderr, "---- TRACE %d stop (%s)\n",
7544 trace_num,
7545 zend_jit_trace_stop_description[stop]);
7546 }
7547 }
7548 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
7549 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
7550 } else {
7551 const zend_op_array *op_array = trace_buffer[0].op_array;
7552 zend_jit_op_array_trace_extension *jit_extension =
7553 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7554 const zend_op *opline = trace_buffer[1].opline;
7555
7556 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
7557 }
7558 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
7559 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
7560 fprintf(stderr, "---- TRACE %d %s\n",
7561 trace_num,
7562 zend_jit_trace_stop_description[stop]);
7563 }
7564 } else {
7565 goto abort;
7566 }
7567 } else {
7568 abort:
7569 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
7570 fprintf(stderr, "---- TRACE %d abort (%s)\n",
7571 trace_num,
7572 zend_jit_trace_stop_description[stop]);
7573 }
7574 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
7575 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
7576 zend_jit_blacklist_trace_exit(parent_num, exit_num);
7577 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7578 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
7579 parent_num, exit_num);
7580 }
7581 }
7582 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
7583 execute_data = EG(current_execute_data);
7584 return zend_jit_trace_hot_root(execute_data, EX(opline));
7585 }
7586 }
7587
7588 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
7589 fprintf(stderr, "\n");
7590 }
7591
7592 return ret;
7593 }
7594
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)7595 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
7596 {
7597 uint32_t trace_num = EG(jit_trace_num);
7598 zend_execute_data *execute_data = EG(current_execute_data);
7599 const zend_op *orig_opline = EX(opline);
7600 const zend_op *opline;
7601 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
7602 int repeat_last_opline = 0;
7603
7604 /* Deoptimizatoion of VM stack state */
7605 uint32_t i;
7606 uint32_t stack_size = t->exit_info[exit_num].stack_size;
7607 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset;
7608
7609 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7610 zend_execute_data *call = (zend_execute_data *)regs->r[ZREG_RX];
7611 call->prev_execute_data = EX(call);
7612 EX(call) = call;
7613 }
7614
7615 for (i = 0; i < stack_size; i++) {
7616 if (STACK_REG(stack, i) != ZREG_NONE) {
7617 if (STACK_TYPE(stack, i) == IS_LONG) {
7618 zend_long val;
7619
7620 if (STACK_REG(stack, i) < ZREG_NUM) {
7621 val = regs->r[STACK_REG(stack, i)];
7622 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN) {
7623 val = ZEND_LONG_MIN;
7624 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX) {
7625 val = ZEND_LONG_MAX;
7626 } else {
7627 ZEND_UNREACHABLE();
7628 }
7629 ZVAL_LONG(EX_VAR_NUM(i), val);
7630 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
7631 double val;
7632
7633 if (STACK_REG(stack, i) < ZREG_NUM) {
7634 val = regs->xmm[STACK_REG(stack, i) - ZREG_XMM0];
7635 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN_MINUS_1) {
7636 val = (double)ZEND_LONG_MIN - 1.0;
7637 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX_PLUS_1) {
7638 val = (double)ZEND_LONG_MAX + 1.0;
7639 } else {
7640 ZEND_UNREACHABLE();
7641 }
7642 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
7643 } else if (STACK_REG(stack, i) == ZREG_THIS) {
7644 zend_object *obj = Z_OBJ(EX(This));
7645
7646 GC_ADDREF(obj);
7647 ZVAL_OBJ(EX_VAR_NUM(i), obj);
7648 } else if (STACK_REG(stack, i) == ZREG_NULL) {
7649 ZVAL_NULL(EX_VAR_NUM(i));
7650 } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) {
7651 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
7652 } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_R0) {
7653 zval *val = (zval*)regs->r[0];
7654
7655 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
7656 /* Undefined array index or property */
7657 repeat_last_opline = 1;
7658 } else {
7659 ZVAL_COPY(EX_VAR_NUM(i), val);
7660 }
7661 } else {
7662 ZEND_UNREACHABLE();
7663 }
7664 }
7665 }
7666
7667 if (repeat_last_opline) {
7668 EX(opline) = t->exit_info[exit_num].opline - 1;
7669 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
7670 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
7671 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
7672 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
7673 }
7674 return 1;
7675 }
7676
7677 opline = t->exit_info[exit_num].opline;
7678
7679 if (opline) {
7680 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
7681 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
7682 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
7683 || (opline-1)->opcode == ZEND_FETCH_LIST_R
7684 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
7685 EX(opline) = opline-1;
7686 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
7687 }
7688 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
7689 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
7690 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
7691 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
7692 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
7693 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
7694 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
7695 EX(opline) = opline-1;
7696 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
7697 }
7698 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
7699 if (EG(exception)) {
7700 return 1;
7701 }
7702 }
7703 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
7704 zend_function *func = (zend_function*)regs->r[0];
7705
7706 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
7707 zend_string_release_ex(func->common.function_name, 0);
7708 zend_free_trampoline(func);
7709 EX(opline) = opline;
7710 return 1;
7711 }
7712 }
7713
7714 /* Set VM opline to continue interpretation */
7715 EX(opline) = opline;
7716 }
7717
7718 if (EG(vm_interrupt) || JIT_G(tracing)) {
7719 return 1;
7720 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
7721 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7722 return 0;
7723 }
7724
7725 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
7726 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
7727 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
7728
7729 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
7730 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
7731 trace_num,
7732 exit_num,
7733 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7734 EX(func)->op_array.scope ? "::" : "",
7735 EX(func)->op_array.function_name ?
7736 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7737 ZSTR_VAL(EX(func)->op_array.filename),
7738 EX(opline)->lineno);
7739 }
7740
7741 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
7742 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
7743 zend_jit_blacklist_trace_exit(trace_num, exit_num);
7744 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7745 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
7746 trace_num, exit_num);
7747 }
7748 return 0;
7749 }
7750 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
7751 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
7752 }
7753
7754 /* Return 1 to call original handler instead of the same JIT-ed trace */
7755 return (orig_opline == t->opline && EX(opline) == orig_opline);
7756 }
7757
zend_jit_trace_supported(const zend_op * opline)7758 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
7759 {
7760 switch (opline->opcode) {
7761 case ZEND_CATCH:
7762 case ZEND_FAST_CALL:
7763 case ZEND_FAST_RET:
7764 return ZEND_JIT_TRACE_UNSUPPORTED;
7765 default:
7766 return ZEND_JIT_TRACE_SUPPORTED;
7767 }
7768 }
7769
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)7770 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
7771 {
7772 zend_jit_op_array_trace_extension *jit_extension;
7773 uint32_t i;
7774
7775 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7776 for (i = 0; i < op_array->last; i++) {
7777 jit_extension->trace_info[i].trace_flags &=
7778 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
7779 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
7780 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
7781 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
7782 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
7783 } else {
7784 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7785 }
7786 }
7787 return SUCCESS;
7788 }
7789
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)7790 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
7791 {
7792 zend_op *opline;
7793 zend_jit_op_array_trace_extension *jit_extension;
7794 uint32_t i;
7795
7796 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
7797
7798 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
7799 if (!jit_extension) {
7800 return FAILURE;
7801 }
7802 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
7803 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
7804 jit_extension->op_array = op_array;
7805 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
7806 for (i = 0; i < op_array->last; i++) {
7807 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
7808 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
7809 jit_extension->trace_info[i].counter = NULL;
7810 jit_extension->trace_info[i].trace_flags =
7811 zend_jit_trace_supported(&op_array->opcodes[i]);
7812 }
7813 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
7814
7815 if (JIT_G(hot_loop)) {
7816 zend_cfg cfg;
7817
7818 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
7819
7820 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
7821 return FAILURE;
7822 }
7823
7824 for (i = 0; i < cfg.blocks_count; i++) {
7825 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
7826 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
7827 /* loop header */
7828 opline = op_array->opcodes + cfg.blocks[i].start;
7829 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
7830 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
7831 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
7832 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
7833 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
7834 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
7835 }
7836 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
7837 ZEND_JIT_TRACE_START_LOOP;
7838 }
7839 }
7840 }
7841 }
7842 }
7843
7844 if (JIT_G(hot_func)) {
7845 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
7846 opline = op_array->opcodes;
7847 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
7848 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
7849 opline++;
7850 }
7851 }
7852
7853 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
7854 /* function entry */
7855 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
7856 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
7857 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
7858 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
7859 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
7860 ZEND_JIT_TRACE_START_ENTER;
7861 }
7862 }
7863
7864 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
7865
7866 return SUCCESS;
7867 }
7868
zend_jit_trace_init_caches(void)7869 static void zend_jit_trace_init_caches(void)
7870 {
7871 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
7872 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
7873 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
7874 JIT_G(bad_root_slot) = 0;
7875
7876 if (JIT_G(exit_counters)) {
7877 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
7878 }
7879 }
7880
zend_jit_trace_reset_caches(void)7881 static void zend_jit_trace_reset_caches(void)
7882 {
7883 JIT_G(tracing) = 0;
7884 #ifdef ZTS
7885 if (!JIT_G(exit_counters)) {
7886 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
7887 }
7888 #endif
7889 }
7890
zend_jit_trace_restart(void)7891 static void zend_jit_trace_restart(void)
7892 {
7893 ZEND_JIT_TRACE_NUM = 1;
7894 ZEND_JIT_COUNTER_NUM = 0;
7895 ZEND_JIT_EXIT_NUM = 0;
7896 ZEND_JIT_EXIT_COUNTERS = 0;
7897
7898 zend_jit_trace_init_caches();
7899 }
7900
