1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(zend_bool reattached)49 static int zend_jit_trace_startup(zend_bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 return FAILURE;
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 return FAILURE;
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 } else {
67 zend_jit_traces = ZCSG(jit_traces);
68 if (!zend_jit_traces) {
69 return FAILURE;
70 }
71 zend_jit_exit_groups = ZCSG(jit_exit_groups);
72 if (!zend_jit_exit_groups) {
73 return FAILURE;
74 }
75 }
76
77 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
78 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
79
80 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
81 if (JIT_G(exit_counters) == NULL) {
82 return FAILURE;
83 }
84
85 return SUCCESS;
86 }
87
zend_jit_trace_allocate_exit_group(uint32_t n)88 static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
89 {
90 dasm_State* dasm_state = NULL;
91 const void *entry;
92 char name[32];
93
94 dasm_init(&dasm_state, DASM_MAXSECTION);
95 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
96 dasm_setup(&dasm_state, dasm_actions);
97 zend_jit_trace_exit_group_stub(&dasm_state, n);
98
99 sprintf(name, "jit$$trace_exit_%d", n);
100 entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE);
101 dasm_free(&dasm_state);
102
103 #ifdef HAVE_DISASM
104 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
105 uint32_t i;
106
107 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
108 sprintf(name, "jit$$trace_exit_%d", n + i);
109 zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
110 }
111 }
112 #endif
113
114 return entry;
115 }
116
zend_jit_trace_allocate_exit_point(uint32_t n)117 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
118 {
119 const void *group = NULL;
120
121 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
122 return NULL;
123 }
124 do {
125 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
126 if (!group) {
127 return NULL;
128 }
129 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
130 group;
131 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
132 } while (n >= ZEND_JIT_EXIT_NUM);
133 return (const void*)
134 ((const char*)group +
135 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
136 }
137
zend_jit_trace_get_exit_addr(uint32_t n)138 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
139 {
140 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
141 return zend_jit_trace_allocate_exit_point(n);
142 }
143 return (const void*)
144 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
145 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
146 }
147
148 #if ZEND_JIT_TARGET_ARM64
zend_jit_get_current_trace_info(void)149 static zend_jit_trace_info *zend_jit_get_current_trace_info(void)
150 {
151 return &zend_jit_traces[ZEND_JIT_TRACE_NUM];
152 }
153
zend_jit_trace_find_exit_point(const void * addr)154 static uint32_t zend_jit_trace_find_exit_point(const void* addr)
155 {
156 uint32_t n = ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP;
157 uint32_t i;
158
159 for (i = 0; i < n; i++) {
160 if ((const char*)addr >= (const char*)zend_jit_exit_groups[i]
161 && (const char*)addr < (const char*)zend_jit_exit_groups[i] +
162 (ZEND_JIT_EXIT_POINTS_PER_GROUP * ZEND_JIT_EXIT_POINTS_SPACING)) {
163 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
164 ((const char*)addr - (const char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING;
165 }
166 }
167 return (uint32_t)-1;
168 }
169 #endif
170
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)171 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
172 {
173 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
174 uint32_t exit_point;
175 const zend_op_array *op_array;
176 uint32_t stack_offset = (uint32_t)-1;
177 uint32_t stack_size;
178 zend_jit_trace_stack *stack = NULL;
179
180 if (delayed_call_chain) {
181 assert(to_opline != NULL); /* CALL and IP share the same register */
182 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
183 }
184 if (JIT_G(current_frame)) {
185 op_array = &JIT_G(current_frame)->func->op_array;
186 stack_size = op_array->last_var + op_array->T;
187 if (stack_size) {
188 stack = JIT_G(current_frame)->stack;
189 do {
190 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
191 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
192 || STACK_REG(stack, stack_size-1) != ZREG_NONE) {
193 break;
194 }
195 stack_size--;
196 } while (stack_size);
197 }
198 } else {
199 op_array = NULL;
200 stack_size = 0;
201 }
202
203 /* Try to reuse exit points */
204 if (to_opline != NULL && t->exit_count > 0) {
205 uint32_t i = t->exit_count;
206
207 do {
208 i--;
209 if (stack_size == 0
210 || (t->exit_info[i].stack_size >= stack_size
211 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
212 stack_offset = t->exit_info[i].stack_offset;
213 if (t->exit_info[i].opline == to_opline
214 && t->exit_info[i].flags == flags
215 && t->exit_info[i].stack_size == stack_size) {
216 return i;
217 }
218 }
219 } while (i > 0);
220 }
221
222 exit_point = t->exit_count;
223 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
224 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
225 stack_offset = t->stack_map_size;
226 t->stack_map_size += stack_size;
227 // TODO: reduce number of reallocations ???
228 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
229 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
230 }
231 t->exit_count++;
232 t->exit_info[exit_point].opline = to_opline;
233 t->exit_info[exit_point].op_array = op_array;
234 t->exit_info[exit_point].flags = flags;
235 t->exit_info[exit_point].stack_size = stack_size;
236 t->exit_info[exit_point].stack_offset = stack_offset;
237 }
238
239 return exit_point;
240 }
241
zend_jit_trace_add_code(const void * start,uint32_t size)242 static void zend_jit_trace_add_code(const void *start, uint32_t size)
243 {
244 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
245
246 t->code_start = start;
247 t->code_size = size;
248 }
249
250 /**
251 * Locate a trace in the #zend_jit_traces array with the specified
252 * #code_start address.
253 *
254 * @return the #zend_jit_traces index or 0 if no such #code_start
255 * address was found
256 */
zend_jit_find_trace(const void * addr)257 static uint32_t zend_jit_find_trace(const void *addr)
258 {
259 uint32_t i;
260
261 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
262 if (zend_jit_traces[i].code_start == addr) {
263 return i;
264 }
265 }
266 return 0;
267 }
268
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)269 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
270 {
271 smart_str buf = {0};
272
273 smart_str_appends(&buf, TRACE_PREFIX);
274 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
275 smart_str_appendc(&buf, '$');
276 if (op_array->function_name) {
277 if (op_array->scope) {
278 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
279 smart_str_appends(&buf, "::");
280 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
281 } else {
282 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
283 }
284 } else if (op_array->filename) {
285 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
286 }
287 smart_str_appendc(&buf, '$');
288 smart_str_append_long(&buf, (zend_long)lineno);
289 smart_str_0(&buf);
290 return buf.s;
291 }
292
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)293 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
294 {
295 switch (opline->opcode) {
296 case ZEND_IS_IDENTICAL:
297 case ZEND_IS_NOT_IDENTICAL:
298 case ZEND_IS_EQUAL:
299 case ZEND_IS_NOT_EQUAL:
300 case ZEND_IS_SMALLER:
301 case ZEND_IS_SMALLER_OR_EQUAL:
302 case ZEND_CASE:
303 case ZEND_CASE_STRICT:
304 case ZEND_ISSET_ISEMPTY_CV:
305 case ZEND_ISSET_ISEMPTY_VAR:
306 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
307 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
308 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
309 case ZEND_INSTANCEOF:
310 case ZEND_TYPE_CHECK:
311 case ZEND_DEFINED:
312 case ZEND_IN_ARRAY:
313 case ZEND_ARRAY_KEY_EXISTS:
314 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
315 /* smart branch */
316 return 1;
317 }
318 break;
319 case ZEND_JMPZNZ:
320 case ZEND_JMPZ:
321 case ZEND_JMPNZ:
322 case ZEND_JMPZ_EX:
323 case ZEND_JMPNZ_EX:
324 case ZEND_JMP_SET:
325 case ZEND_COALESCE:
326 case ZEND_JMP_NULL:
327 case ZEND_FE_RESET_R:
328 case ZEND_FE_RESET_RW:
329 case ZEND_ASSERT_CHECK:
330 case ZEND_FE_FETCH_R:
331 case ZEND_FE_FETCH_RW:
332 case ZEND_SWITCH_LONG:
333 case ZEND_SWITCH_STRING:
334 case ZEND_MATCH:
335 /* branch opcodes */
336 return 1;
337 case ZEND_NEW:
338 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
339 /* NEW may skip constructor without arguments */
340 return 1;
341 }
342 break;
343 case ZEND_CATCH:
344 case ZEND_FAST_CALL:
345 case ZEND_FAST_RET:
346 case ZEND_GENERATOR_CREATE:
347 case ZEND_GENERATOR_RETURN:
348 case ZEND_EXIT:
349 case ZEND_YIELD:
350 case ZEND_YIELD_FROM:
351 case ZEND_INCLUDE_OR_EVAL:
352 case ZEND_MATCH_ERROR:
353 /* unsupported */
354 return 1;
355 case ZEND_DO_FCALL:
356 /* potentially polymorphic call */
357 return 1;
358 #if 0
359 case ZEND_DO_UCALL:
360 case ZEND_DO_FCALL_BY_NAME:
361 /* monomorphic call */
362 // TODO: recompilation may change target ???
363 return 0;
364 #endif
365 case ZEND_RETURN_BY_REF:
366 case ZEND_RETURN:
367 /* return */
368 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
369 default:
370 break;
371 }
372 return 0;
373 }
374
zend_jit_trace_type_to_info_ex(zend_uchar type,uint32_t info)375 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(zend_uchar type, uint32_t info)
376 {
377 if (type == IS_UNKNOWN) {
378 return info;
379 }
380 ZEND_ASSERT(info & (1 << type));
381 if (type < IS_STRING) {
382 return (1 << type);
383 } else if (type != IS_ARRAY) {
384 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
385 } else {
386 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
387 }
388 }
389
zend_jit_trace_type_to_info(zend_uchar type)390 static zend_always_inline uint32_t zend_jit_trace_type_to_info(zend_uchar type)
391 {
392 return zend_jit_trace_type_to_info_ex(type, -1);
393 }
394
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)395 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
396 {
397 if (var >= op_array->last_var) {
398 return NO_ALIAS;
399 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
400 return SYMTABLE_ALIAS;
401 } else if (ssa->vars) {
402 return ssa->vars[var].alias;
403 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
404 return HTTP_RESPONSE_HEADER_ALIAS;
405 }
406 return NO_ALIAS;
407 }
408
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)409 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
410 int ssa_var,
411 uint8_t op_type)
412 {
413 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
414
415 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
416 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
417 info->type |= MAY_BE_GUARD;
418 } else {
419 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
420 }
421 }
422 }
423
424 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
425 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
426 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
427 } \
428 } while (0)
429
430 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
431 if (op_type != IS_UNKNOWN) { \
432 if ((op_info & MAY_BE_GUARD) != 0) { \
433 if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \
434 goto jit_failure; \
435 } \
436 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
437 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
438 op_info = zend_jit_trace_type_to_info(op_type); \
439 } else { \
440 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
441 op_info &= ~MAY_BE_GUARD; \
442 ssa->var_info[_ssa_var].type &= op_info; \
443 } \
444 } \
445 } \
446 } while (0)
447
448 #define ADD_OP1_TRACE_GUARD() \
449 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
450 #define ADD_OP2_TRACE_GUARD() \
451 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
452 #define ADD_OP1_DATA_TRACE_GUARD() \
453 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
454
455 #define CHECK_OP1_TRACE_TYPE() \
456 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
457 #define CHECK_OP2_TRACE_TYPE() \
458 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
459 #define CHECK_OP1_DATA_TRACE_TYPE() \
460 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
461
zend_jit_trace_frame_size(const zend_op_array * op_array)462 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
463 {
464 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
465 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
466 } else if (op_array) {
467 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
468 } else {
469 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
470 }
471 }
472
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)473 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
474 {
475 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
476 }
477
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)478 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
479 {
480 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
481 }
482
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,zend_uchar type)483 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, zend_uchar type)
484 {
485 zend_jit_trace_stack *stack = call->stack;
486 const zend_op_array *op_array = &call->func->op_array;
487 uint32_t arg_num = opline->op2.num;
488
489 if (arg_num > op_array->num_args) {
490 return;
491 }
492 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
493 zend_arg_info *arg_info;
494
495 ZEND_ASSERT(arg_num <= op_array->num_args);
496 arg_info = &op_array->arg_info[arg_num-1];
497
498 if (ZEND_TYPE_IS_SET(arg_info->type)) {
499 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
500 return;
501 }
502 }
503 }
504 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
505 }
506
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)507 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
508 {
509 if (func
510 && func->type == ZEND_INTERNAL_FUNCTION
511 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
512 && arg_num < func->internal_function.num_args) {
513 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
514
515 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
516 && ZEND_TYPE_IS_SET(arg_info->type)
517 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
518 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
519 && JIT_G(current_frame)
520 && JIT_G(current_frame)->call
521 && JIT_G(current_frame)->call->func) {
522 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
523
524 if (type != IS_UNKNOWN
525 && type < IS_STRING
526 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
527 return 0;
528 }
529 }
530 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
531 const zend_op *opline = call_info->arg_info[arg_num].opline;
532
533 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
534 zval *zv = RT_CONSTANT(opline, opline->op1);
535
536 if (!Z_REFCOUNTED_P(zv)) {
537 uint32_t type = Z_TYPE_P(zv);
538
539 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
540 if (type != IS_ARRAY
541 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
542 return 0;
543 }
544 }
545 }
546 }
547 }
548 }
549
550 return 1;
551 }
552
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)553 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
554 {
555 zend_jit_op_array_trace_extension *jit_extension;
556 zend_ssa *ssa;
557
558 jit_extension =
559 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
560 jit_extension->func_info.num = 0;
561 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
562 | ZEND_FUNC_JIT_ON_PROF_REQUEST
563 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
564 | ZEND_FUNC_JIT_ON_HOT_TRACE;
565 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
566 ssa = &jit_extension->func_info.ssa;
567
568 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
569 do {
570 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
571 break;
572 }
573
574 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
575 if (zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info) != SUCCESS) {
576 break;
577 }
578 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
579 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
580 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
581 }
582 }
583
584 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
585 break;
586 }
587
588 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
589 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
590 }
591 return ssa;
592 } while (0);
593 }
594
595 memset(ssa, 0, sizeof(zend_ssa));
596 ssa->cfg.blocks_count = 1;
597
598 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
599 zend_cfg cfg;
600 void *checkpoint = zend_arena_checkpoint(CG(arena));
601
602 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
603 ssa->cfg.flags = cfg.flags;
604 } else{
605 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
606 }
607
608 /* TODO: move this to zend_cfg.c ? */
609 if (!op_array->function_name) {
610 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
611 }
612
613 zend_arena_release(&CG(arena), checkpoint);
614 }
615
616 return ssa;
617 }
618
619 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
620 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
621
zend_jit_trace_op_len(const zend_op * opline)622 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
623 {
624 int len;
625
626 switch (opline->opcode) {
627 case ZEND_ASSIGN_DIM:
628 case ZEND_ASSIGN_OBJ:
629 case ZEND_ASSIGN_STATIC_PROP:
630 case ZEND_ASSIGN_DIM_OP:
631 case ZEND_ASSIGN_OBJ_OP:
632 case ZEND_ASSIGN_STATIC_PROP_OP:
633 case ZEND_ASSIGN_OBJ_REF:
634 case ZEND_ASSIGN_STATIC_PROP_REF:
635 return 2; /* OP_DATA */
636 case ZEND_RECV_INIT:
637 len = 1;
638 opline++;
639 while (opline->opcode == ZEND_RECV_INIT) {
640 len++;
641 opline++;
642 }
643 return len;
644 case ZEND_BIND_GLOBAL:
645 len = 1;
646 opline++;
647 while (opline->opcode == ZEND_BIND_GLOBAL) {
648 len++;
649 opline++;
650 }
651 return len;
652 // case ZEND_IS_IDENTICAL:
653 // case ZEND_IS_NOT_IDENTICAL:
654 // case ZEND_IS_EQUAL:
655 // case ZEND_IS_NOT_EQUAL:
656 // case ZEND_IS_SMALLER:
657 // case ZEND_IS_SMALLER_OR_EQUAL:
658 // case ZEND_CASE:
659 // case ZEND_ISSET_ISEMPTY_CV:
660 // case ZEND_ISSET_ISEMPTY_VAR:
661 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
662 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
663 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
664 // case ZEND_INSTANCEOF:
665 // case ZEND_TYPE_CHECK:
666 // case ZEND_DEFINED:
667 // case ZEND_IN_ARRAY:
668 // case ZEND_ARRAY_KEY_EXISTS:
669 default:
670 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
671 return 2; /* JMPZ/JMPNZ */
672 }
673 return 1;
674 }
675 }
676
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)677 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
678 {
679 const zend_op_array *op_array;
680 zend_jit_trace_rec *p;
681 int k, vars_count;
682 zend_bitset use, def;
683 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
684 uint32_t set_size;
685 zend_ssa_phi *prev = NULL;
686 int level = 0;
687 ALLOCA_FLAG(use_heap);
688
689 op_array = trace_buffer->op_array;
690 set_size = zend_bitset_len(op_array->last_var + op_array->T);
691 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
692 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
693 def = use + set_size;
694 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
695 for (;;p++) {
696 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
697 const zend_op *opline = p->opline;
698 int len;
699
700 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
701 len = zend_jit_trace_op_len(opline);
702 while (len > 1) {
703 opline++;
704 if (opline->opcode != ZEND_OP_DATA) {
705 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
706 }
707 len--;
708 }
709 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
710 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
711 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
712 level++;
713 } else if (p->op == ZEND_JIT_TRACE_BACK) {
714 if (level == 0) {
715 // Phi for recursive calls and returns are not supported yet ???
716 assert(0);
717 } else {
718 level--;
719 }
720 } else if (p->op == ZEND_JIT_TRACE_END) {
721 break;
722 }
723 }
724
725 zend_bitset_intersection(use, def, set_size);
726
727 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
728 vars_count = op_array->last_var;
729 } else {
730 vars_count = op_array->last_var + op_array->T;
731 }
732 for (k = 0; k < vars_count; k++) {
733 if (zend_bitset_in(use, k)) {
734 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
735 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
736 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
737 sizeof(void*) * 2);
738 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
739 phi->sources[0] = STACK_VAR(stack, k);
740 phi->sources[1] = -1;
741 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
742 phi->pi = -1;
743 phi->var = k;
744 phi->ssa_var = ssa_vars_count;
745 SET_STACK_VAR(stack, k, ssa_vars_count);
746 ssa_vars_count++;
747 phi->block = 1;
748 if (prev) {
749 prev->next = phi;
750 } else {
751 tssa->blocks[1].phis = phi;
752 }
753 prev = phi;
754 }
755 }
756
757 free_alloca(use, use_heap);
758
759 return ssa_vars_count;
760 }
761
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)762 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
763 {
764 zend_ssa_phi *prev = NULL;
765 const zend_op_array *op_array = trace_buffer->op_array;
766 const zend_op *opline = trace_buffer[1].opline;
767 int count = opline - op_array->opcodes;
768 int i;
769
770 for(i = 0; i < count; i++) {
771 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
772 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
773 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
774 sizeof(void*) * 2);
775 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
776 phi->sources[0] = STACK_VAR(stack, i);
777 phi->sources[1] = -1;
778 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
779 phi->pi = -1;
780 phi->var = i;
781 phi->ssa_var = ssa_vars_count;
782 SET_STACK_VAR(stack, i, ssa_vars_count);
783 ssa_vars_count++;
784 phi->block = 1;
785 if (prev) {
786 prev->next = phi;
787 } else {
788 tssa->blocks[1].phis = phi;
789 }
790 prev = phi;
791 }
792 return ssa_vars_count;
793 }
794
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)795 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
796 {
797 const zend_op *opline = trace_buffer[1].opline - 1;
798 int i;
799
800 if (RETURN_VALUE_USED(opline)) {
801 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
802 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
803 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
804 sizeof(void*) * 2);
805
806 i = EX_VAR_TO_NUM(opline->result.var);
807 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
808 phi->sources[0] = STACK_VAR(stack, i);
809 phi->sources[1] = -1;
810 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
811 phi->pi = -1;
812 phi->var = i;
813 phi->ssa_var = ssa_vars_count;
814 SET_STACK_VAR(stack, i, ssa_vars_count);
815 ssa_vars_count++;
816 phi->block = 1;
817 tssa->blocks[1].phis = phi;
818 }
819 return ssa_vars_count;
820 }
821
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)822 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
823 {
824 int var, use;
825 zend_ssa_op *op;
826 zend_ssa_var_info *info;
827 unsigned int no_val;
828 zend_ssa_alias_kind alias;
829
830 if (tssa->vars[ssa_var].phi_use_chain) {
831 // TODO: this may be incorrect ???
832 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
833 } else {
834 var = ssa_var;
835 }
836 use = tssa->vars[var].use_chain;
837 if (use >= 0) {
838 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
839 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
840 if (tssa->ops[use].op1_use == var) {
841 no_val = ssa->vars[op->op1_use].no_val;
842 alias = ssa->vars[op->op1_use].alias;
843 info = ssa->var_info + op->op1_use;
844 } else if (tssa->ops[use].op2_use == var) {
845 no_val = ssa->vars[op->op2_use].no_val;
846 alias = ssa->vars[op->op2_use].alias;
847 info = ssa->var_info + op->op2_use;
848 } else if (tssa->ops[use].result_use == var) {
849 no_val = ssa->vars[op->result_use].no_val;
850 alias = ssa->vars[op->result_use].alias;
851 info = ssa->var_info + op->result_use;
852 } else {
853 assert(0);
854 return 0;
855 }
856 tssa->vars[ssa_var].no_val = no_val;
857 tssa->vars[ssa_var].alias = alias;
858 memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info));
859 return 1;
860 }
861 return 0;
862 }
863
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)864 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
865 {
866 zend_ssa_range tmp;
867 int def = tssa->vars[ssa_var].definition;
868
869 if (tssa->vars[ssa_var].alias == NO_ALIAS
870 && zend_inference_propagate_range(op_array, tssa, (zend_op*)tssa_opcodes[def], (zend_ssa_op*)&tssa->ops[def], ssa_var, &tmp)) {
871 tssa->var_info[ssa_var].range.min = tmp.min;
872 tssa->var_info[ssa_var].range.max = tmp.max;
873 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
874 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
875 tssa->var_info[ssa_var].has_range = 1;
876 }
877 }
878
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)879 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
880 {
881 int def;
882 zend_ssa_op *op;
883 zend_ssa_var_info *info;
884 unsigned int no_val;
885 zend_ssa_alias_kind alias;
886
887 def = tssa->vars[ssa_var].definition;
888 if (def >= 0) {
889 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
890 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
891 if (tssa->ops[def].op1_def == ssa_var) {
892 no_val = ssa->vars[op->op1_def].no_val;
893 alias = ssa->vars[op->op1_def].alias;
894 info = ssa->var_info + op->op1_def;
895 } else if (tssa->ops[def].op2_def == ssa_var) {
896 no_val = ssa->vars[op->op2_def].no_val;
897 alias = ssa->vars[op->op2_def].alias;
898 info = ssa->var_info + op->op2_def;
899 } else if (tssa->ops[def].result_def == ssa_var) {
900 no_val = ssa->vars[op->result_def].no_val;
901 alias = ssa->vars[op->result_def].alias;
902 info = ssa->var_info + op->result_def;
903 } else {
904 assert(0);
905 return;
906 }
907
908 tssa->vars[ssa_var].no_val = no_val;
909 tssa->vars[ssa_var].alias = alias;
910
911 if (!(info->type & MAY_BE_REF)) {
912 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
913 }
914
915 if (info->has_range) {
916 if (tssa->var_info[ssa_var].has_range) {
917 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
918 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
919 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
920 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
921 } else {
922 tssa->var_info[ssa_var].has_range = 1;
923 tssa->var_info[ssa_var].range = info->range;
924 }
925 }
926 }
927 }
928
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)929 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
930 {
931 int def;
932 zend_ssa_op *op;
933 zend_ssa_var_info *info;
934
935 def = tssa->vars[ssa_var].definition;
936 if (def >= 0) {
937 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
938 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
939 if (tssa->ops[def].op1_def == ssa_var) {
940 info = ssa->var_info + op->op1_def;
941 } else if (tssa->ops[def].op2_def == ssa_var) {
942 info = ssa->var_info + op->op2_def;
943 } else if (tssa->ops[def].result_def == ssa_var) {
944 info = ssa->var_info + op->result_def;
945 } else {
946 assert(0);
947 return 0;
948 }
949 tssa->var_info[ssa_var].type &= info->type;
950 if (info->ce) {
951 if (tssa->var_info[ssa_var].ce) {
952 if (tssa->var_info[ssa_var].ce != info->ce) {
953 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
954 /* everything fine */
955 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
956 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
957 } else {
958 // TODO: classes may implement the same interface ???
959 //ZEND_UNREACHABLE();
960 }
961 }
962 tssa->var_info[ssa_var].is_instanceof =
963 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
964 } else {
965 tssa->var_info[ssa_var].ce = info->ce;
966 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
967 }
968 }
969 if (info->has_range) {
970 if (tssa->var_info[ssa_var].has_range) {
971 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
972 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
973 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
974 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
975 } else {
976 tssa->var_info[ssa_var].has_range = 1;
977 tssa->var_info[ssa_var].range = info->range;
978 }
979 }
980 return 1;
981 }
982 return 0;
983 }
984
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)985 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
986 {
987 while (1) {
988 if (p->op == ZEND_JIT_TRACE_VM) {
989 if (p->opline->opcode == ZEND_DO_UCALL
990 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
991 || p->opline->opcode == ZEND_DO_FCALL) {
992 if (p->opline->result_type != IS_UNUSED) {
993 return ssa_op->result_def;
994 }
995 }
996 return -1;
997 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
998 /*skip */
999 } else {
1000 return -1;
1001 }
1002 p--;
1003 }
1004 }
1005
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1006 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1007 {
1008 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1009 p--;
1010 while (1) {
1011 if (p->op == ZEND_JIT_TRACE_VM) {
1012 if (p->opline->opcode == ZEND_INIT_FCALL
1013 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1014 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1015 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1016 || p->opline->opcode == ZEND_INIT_USER_CALL
1017 || p->opline->opcode == ZEND_NEW
1018 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1019 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
1020 return p->opline;
1021 }
1022 return NULL;
1023 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1024 /*skip */
1025 } else {
1026 return NULL;
1027 }
1028 p--;
1029 }
1030 } else {
1031 const zend_op *opline = NULL;
1032 int call_level = 0;
1033
1034 p++;
1035 while (1) {
1036 if (p->op == ZEND_JIT_TRACE_VM) {
1037 opline = p->opline;
1038 break;
1039 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1040 call_level++;
1041 /*skip */
1042 } else {
1043 return NULL;
1044 }
1045 p--;
1046 }
1047 if (opline) {
1048 while (opline > op_array->opcodes) {
1049 opline--;
1050 switch (opline->opcode) {
1051 case ZEND_INIT_FCALL:
1052 case ZEND_INIT_FCALL_BY_NAME:
1053 case ZEND_INIT_NS_FCALL_BY_NAME:
1054 case ZEND_INIT_METHOD_CALL:
1055 case ZEND_INIT_DYNAMIC_CALL:
1056 case ZEND_INIT_STATIC_METHOD_CALL:
1057 case ZEND_INIT_USER_CALL:
1058 case ZEND_NEW:
1059 if (call_level == 0) {
1060 return opline;
1061 }
1062 call_level--;
1063 break;
1064 case ZEND_DO_FCALL:
1065 case ZEND_DO_ICALL:
1066 case ZEND_DO_UCALL:
1067 case ZEND_DO_FCALL_BY_NAME:
1068 case ZEND_CALLABLE_CONVERT:
1069 call_level++;
1070 break;
1071 }
1072 }
1073 }
1074 }
1075 return NULL;
1076 }
1077
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1078 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1079 {
1080 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1081 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1082 int idx = tssa->vars[var].definition;
1083
1084 if (idx >= 0) {
1085 if (tssa->ops[idx].op1_def == var) {
1086 const zend_op *opline = ssa_opcodes[idx];
1087 if (opline->opcode == ZEND_PRE_DEC
1088 || opline->opcode == ZEND_PRE_INC
1089 || opline->opcode == ZEND_POST_DEC
1090 || opline->opcode == ZEND_POST_INC) {
1091 if (tssa->ops[idx].op1_use >= 0
1092 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1093 return 0;
1094 }
1095 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1096 return 0;
1097 }
1098 return 1;
1099 } else if (opline->opcode == ZEND_ASSIGN_OP
1100 && (opline->extended_value == ZEND_ADD
1101 || opline->extended_value == ZEND_SUB
1102 || opline->extended_value == ZEND_MUL)) {
1103 if ((opline->op2_type & (IS_VAR|IS_CV))
1104 && tssa->ops[idx].op2_use >= 0
1105 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1106 return 0;
1107 }
1108 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1109 return 0;
1110 }
1111 if (opline->op2_type == IS_CONST) {
1112 zval *zv = RT_CONSTANT(opline, opline->op2);
1113 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1114 return 0;
1115 }
1116 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1117 return 0;
1118 }
1119 return 1;
1120 }
1121 }
1122 if (tssa->ops[idx].result_def == var) {
1123 const zend_op *opline = ssa_opcodes[idx];
1124 if (opline->opcode == ZEND_ADD
1125 || opline->opcode == ZEND_SUB
1126 || opline->opcode == ZEND_MUL) {
1127 if ((opline->op1_type & (IS_VAR|IS_CV))
1128 && tssa->ops[idx].op1_use >= 0
1129 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1130 return 0;
1131 }
1132 if ((opline->op2_type & (IS_VAR|IS_CV))
1133 && tssa->ops[idx].op2_use >= 0
1134 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1135 return 0;
1136 }
1137 if (opline->op1_type == IS_CONST) {
1138 zval *zv = RT_CONSTANT(opline, opline->op1);
1139 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1140 return 0;
1141 }
1142 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1143 return 0;
1144 }
1145 if (opline->op2_type == IS_CONST) {
1146 zval *zv = RT_CONSTANT(opline, opline->op2);
1147 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1148 return 0;
1149 }
1150 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1151 return 0;
1152 }
1153 } else if (opline->opcode == ZEND_PRE_DEC
1154 || opline->opcode == ZEND_PRE_INC
1155 || opline->opcode == ZEND_POST_DEC
1156 || opline->opcode == ZEND_POST_INC) {
1157 if ((opline->op1_type & (IS_VAR|IS_CV))
1158 && tssa->ops[idx].op1_use >= 0
1159 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1160 return 0;
1161 }
1162 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1163 return 0;
1164 }
1165 return 1;
1166 }
1167 }
1168 }
1169 }
1170 return 0;
1171 }
1172
1173 typedef struct _zend_tssa {
1174 zend_ssa ssa;
1175 const zend_op **tssa_opcodes;
1176 int used_stack;
1177 } zend_tssa;
1178
1179 static const zend_op _nop_opcode = {0};
1180
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1181 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1182 {
1183 zend_ssa *tssa;
1184 zend_ssa_op *ssa_ops, *op;
1185 zend_ssa_var *ssa_vars;
1186 zend_ssa_var_info *ssa_var_info;
1187 const zend_op_array *op_array;
1188 const zend_op *opline;
1189 const zend_op **ssa_opcodes;
1190 zend_jit_trace_rec *p;
1191 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1192 zend_jit_trace_stack *stack;
1193 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1194 uint32_t optimization_level = 0;
1195 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1196 size_t frame_size, stack_top, stack_size, stack_bottom;
1197 zend_jit_op_array_trace_extension *jit_extension;
1198 zend_ssa *ssa;
1199 zend_jit_trace_stack_frame *frame, *top, *call;
1200 zend_ssa_var_info return_value_info;
1201
1202 /* 1. Count number of TSSA opcodes;
1203 * Count number of activation frames;
1204 * Calculate size of abstract stack;
1205 * Construct regular SSA for involved op_array */
1206 op_array = trace_buffer->op_array;
1207 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1208 stack_bottom = 0;
1209 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1210 ssa_ops_count = 0;
1211 call_level = 0;
1212 level = 0;
1213 num_op_arrays = 0;
1214 /* Remember op_array to cleanup */
1215 op_arrays[num_op_arrays++] = op_array;
1216 /* Build SSA */
1217 ssa = zend_jit_trace_build_ssa(op_array, script);
1218 for (;;p++) {
1219 if (p->op == ZEND_JIT_TRACE_VM) {
1220 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1221 const zend_op *opline = p->opline;
1222
1223 switch (opline->opcode) {
1224 case ZEND_INCLUDE_OR_EVAL:
1225 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1226 break;
1227 case ZEND_FETCH_R:
1228 case ZEND_FETCH_W:
1229 case ZEND_FETCH_RW:
1230 case ZEND_FETCH_FUNC_ARG:
1231 case ZEND_FETCH_IS:
1232 case ZEND_FETCH_UNSET:
1233 case ZEND_UNSET_VAR:
1234 case ZEND_ISSET_ISEMPTY_VAR:
1235 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1236 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1237 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1238 !op_array->function_name) {
1239 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1240 }
1241 break;
1242 }
1243 }
1244 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1245 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1246 call_level++;
1247 stack_top += zend_jit_trace_frame_size(p->op_array);
1248 if (stack_top > stack_size) {
1249 stack_size = stack_top;
1250 }
1251 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1252 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1253 if (p->func != (zend_function*)&zend_pass_function
1254 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1255 || zend_string_equals_literal(p->func->common.function_name, "compact")
1256 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1257 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1258 }
1259 }
1260 frame_size = zend_jit_trace_frame_size(p->op_array);
1261 if (call_level == 0) {
1262 if (stack_top + frame_size > stack_size) {
1263 stack_size = stack_top + frame_size;
1264 }
1265 } else {
1266 call_level--;
1267 stack_top -= frame_size;
1268 }
1269 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1270 op_array = p->op_array;
1271 if (call_level == 0) {
1272 stack_top += zend_jit_trace_frame_size(op_array);
1273 if (stack_top > stack_size) {
1274 stack_size = stack_top;
1275 }
1276 } else {
1277 call_level--;
1278 }
1279 level++;
1280 jit_extension =
1281 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1282 ssa = &jit_extension->func_info.ssa;
1283 if (ssa->cfg.blocks_count) {
1284 /* pass */
1285 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1286 /* Too many functions in single trace */
1287 *num_op_arrays_ptr = num_op_arrays;
1288 return NULL;
1289 } else {
1290 /* Remember op_array to cleanup */
1291 op_arrays[num_op_arrays++] = op_array;
1292 /* Build SSA */
1293 ssa = zend_jit_trace_build_ssa(op_array, script);
1294 }
1295 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1296 if (level == 0) {
1297 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1298 jit_extension =
1299 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1300 ssa = &jit_extension->func_info.ssa;
1301 if (ssa->cfg.blocks_count) {
1302 /* pass */
1303 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1304 /* Too many functions in single trace */
1305 *num_op_arrays_ptr = num_op_arrays;
1306 return NULL;
1307 } else {
1308 /* Remember op_array to cleanup */
1309 op_arrays[num_op_arrays++] = op_array;
1310 /* Build SSA */
1311 ssa = zend_jit_trace_build_ssa(op_array, script);
1312 }
1313 } else {
1314 stack_top -= zend_jit_trace_frame_size(op_array);
1315 level--;
1316 }
1317 op_array = p->op_array;
1318 } else if (p->op == ZEND_JIT_TRACE_END) {
1319 break;
1320 }
1321 }
1322 *num_op_arrays_ptr = num_op_arrays;
1323
1324 /* Allocate space for abstract stack */
1325 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1326
1327 /* 2. Construct TSSA */
1328 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1329 tssa->cfg.flags = ZEND_SSA_TSSA;
1330 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1331 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1332 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1333
1334 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1335 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1336 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1337 tssa->cfg.blocks_count = 2;
1338 tssa->cfg.edges_count = 2;
1339
1340 tssa->cfg.predecessors[0] = 0;
1341 tssa->cfg.predecessors[1] = 1;
1342
1343 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1344 tssa->cfg.blocks[0].successors_count = 1;
1345 tssa->cfg.blocks[0].predecessors_count = 0;
1346 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1347 tssa->cfg.blocks[0].successors[0] = 1;
1348
1349 tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1350 tssa->cfg.blocks[1].successors_count = 1;
1351 tssa->cfg.blocks[1].predecessors_count = 2;
1352 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1353 tssa->cfg.blocks[1].successors[1] = 1;
1354 } else {
1355 tssa->cfg.blocks_count = 1;
1356 tssa->cfg.edges_count = 0;
1357
1358 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1359 tssa->cfg.blocks[0].successors_count = 0;
1360 tssa->cfg.blocks[0].predecessors_count = 0;
1361 }
1362 ((zend_tssa*)tssa)->used_stack = -1;
1363
1364 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1365 return tssa;
1366 }
1367
1368 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1369 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1370 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1371 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1372 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1373
1374 op_array = trace_buffer->op_array;
1375 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1376 ssa_vars_count = op_array->last_var;
1377 } else {
1378 ssa_vars_count = op_array->last_var + op_array->T;
1379 }
1380 stack = frame->stack;
1381 for (i = 0; i < ssa_vars_count; i++) {
1382 SET_STACK_VAR(stack, i, i);
1383 }
1384
1385 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1386 // TODO: For tracing, it's possible, to create pseudo Phi functions
1387 // at the end of loop, without this additional pass (like LuaJIT) ???
1388 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1389 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1390 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1391 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1392 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1393 }
1394
1395 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1396 idx = 0;
1397 level = 0;
1398 for (;;p++) {
1399 if (p->op == ZEND_JIT_TRACE_VM) {
1400 opline = p->opline;
1401 ssa_opcodes[idx] = opline;
1402 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1403 idx++;
1404 len = zend_jit_trace_op_len(p->opline);
1405 while (len > 1) {
1406 opline++;
1407 ssa_opcodes[idx] = opline;
1408 if (opline->opcode != ZEND_OP_DATA) {
1409 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1410 }
1411 idx++;
1412 len--;
1413 }
1414 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1415 frame = zend_jit_trace_call_frame(frame, op_array);
1416 stack = frame->stack;
1417 op_array = p->op_array;
1418 level++;
1419 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1420 return NULL;
1421 }
1422 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1423 for (i = 0; i < op_array->last_var; i++) {
1424 SET_STACK_VAR(stack, i, ssa_vars_count++);
1425 }
1426 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1427 op_array = p->op_array;
1428 frame = zend_jit_trace_ret_frame(frame, op_array);
1429 stack = frame->stack;
1430 if (level == 0) {
1431 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1432 return NULL;
1433 }
1434 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1435 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1436 SET_STACK_VAR(stack, i, ssa_vars_count++);
1437 }
1438 } else {
1439 level--;
1440 }
1441 } else if (p->op == ZEND_JIT_TRACE_END) {
1442 break;
1443 }
1444 }
1445
1446 op_array = trace_buffer->op_array;
1447 tssa->vars_count = ssa_vars_count;
1448 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1449 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1450 vars_count = op_array->last_var;
1451 } else {
1452 vars_count = op_array->last_var + op_array->T;
1453 }
1454 i = 0;
1455 while (i < vars_count) {
1456 ssa_vars[i].var = i;
1457 ssa_vars[i].scc = -1;
1458 ssa_vars[i].definition = -1;
1459 ssa_vars[i].use_chain = -1;
1460 i++;
1461 }
1462 while (i < tssa->vars_count) {
1463 ssa_vars[i].var = -1;
1464 ssa_vars[i].scc = -1;
1465 ssa_vars[i].definition = -1;
1466 ssa_vars[i].use_chain = -1;
1467 i++;
1468 }
1469
1470 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1471 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1472 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1473 /* Update Phi sources */
1474 zend_ssa_phi *phi = tssa->blocks[1].phis;
1475
1476 while (phi) {
1477 phi->sources[1] = STACK_VAR(stack, phi->var);
1478 ssa_vars[phi->ssa_var].var = phi->var;
1479 ssa_vars[phi->ssa_var].definition_phi = phi;
1480 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1481 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1482 phi = phi->next;
1483 }
1484 }
1485
1486 /* 3. Compute use-def chains */
1487 idx = (ssa_ops_count - 1);
1488 op = ssa_ops + idx;
1489 while (idx >= 0) {
1490 opline = ssa_opcodes[idx];
1491 if (op->op1_use >= 0) {
1492 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1493 ssa_vars[op->op1_use].use_chain = idx;
1494 }
1495 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1496 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1497 ssa_vars[op->op2_use].use_chain = idx;
1498 }
1499 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1500 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1501 ssa_vars[op->result_use].use_chain = idx;
1502 }
1503 if (op->op1_def >= 0) {
1504 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1505 ssa_vars[op->op1_def].definition = idx;
1506 }
1507 if (op->op2_def >= 0) {
1508 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1509 ssa_vars[op->op2_def].definition = idx;
1510 }
1511 if (op->result_def >= 0) {
1512 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1513 ssa_vars[op->result_def].definition = idx;
1514 }
1515 op--;
1516 idx--;
1517 }
1518
1519 /* 4. Type inference */
1520 op_array = trace_buffer->op_array;
1521 jit_extension =
1522 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1523 ssa = &jit_extension->func_info.ssa;
1524
1525 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1526
1527 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1528 i = 0;
1529 while (i < op_array->last_var) {
1530 if (i < op_array->num_args) {
1531 if (ssa->var_info
1532 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1533 /* pass */
1534 } else {
1535 if (ssa->vars) {
1536 ssa_vars[i].no_val = ssa->vars[i].no_val;
1537 ssa_vars[i].alias = ssa->vars[i].alias;
1538 } else {
1539 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1540 }
1541 if (op_array->arg_info) {
1542 zend_arg_info *arg_info = &op_array->arg_info[i];
1543 zend_class_entry *ce;
1544 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1545
1546 if (ZEND_ARG_SEND_MODE(arg_info)) {
1547 tmp |= MAY_BE_REF;
1548 }
1549 ssa_var_info[i].type = tmp;
1550 ssa_var_info[i].ce = ce;
1551 ssa_var_info[i].is_instanceof = 1;
1552 } else {
1553 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1554 }
1555 }
1556 } else {
1557 if (ssa->vars) {
1558 ssa_vars[i].no_val = ssa->vars[i].no_val;
1559 ssa_vars[i].alias = ssa->vars[i].alias;
1560 } else {
1561 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1562 }
1563 if (ssa_vars[i].alias == NO_ALIAS) {
1564 ssa_var_info[i].type = MAY_BE_UNDEF;
1565 } else {
1566 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1567 }
1568 }
1569 i++;
1570 }
1571 } else {
1572 int parent_vars_count = 0;
1573 zend_jit_trace_stack *parent_stack = NULL;
1574
1575 i = 0;
1576 if (parent_trace) {
1577 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1578 op_array->last_var + op_array->T);
1579 if (parent_vars_count) {
1580 parent_stack =
1581 zend_jit_traces[parent_trace].stack_map +
1582 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1583 }
1584 }
1585 while (i < op_array->last_var + op_array->T) {
1586 if (!ssa->var_info
1587 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1588 if (ssa->vars && i < ssa->vars_count) {
1589 ssa_vars[i].alias = ssa->vars[i].alias;
1590 } else {
1591 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1592 }
1593 if (i < op_array->last_var) {
1594 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1595 } else {
1596 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1597 }
1598 }
1599 if (i < parent_vars_count) {
1600 /* Initialize TSSA variable from parent trace */
1601 zend_uchar op_type = STACK_TYPE(parent_stack, i);
1602
1603 if (op_type != IS_UNKNOWN) {
1604 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1605 if (!ssa_var_info[i].type
1606 && op_type == IS_UNDEF
1607 && i >= op_array->last_var) {
1608 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1609 ssa_var_info[i].type |= MAY_BE_UNDEF;
1610 }
1611 }
1612 }
1613 i++;
1614 }
1615 }
1616
1617 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1618 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1619 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1620 /* Propagate initial value through Phi functions */
1621 zend_ssa_phi *phi = tssa->blocks[1].phis;
1622
1623 while (phi) {
1624 if (!ssa->var_info
1625 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) {
1626 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1627 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1628 }
1629 phi = phi->next;
1630 }
1631 }
1632
1633 frame = JIT_G(current_frame);
1634 top = zend_jit_trace_call_frame(frame, op_array);
1635 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1636 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1637 frame->used_stack = 0;
1638 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1639 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
1640 }
1641 memset(&return_value_info, 0, sizeof(return_value_info));
1642
1643 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1644 max_used_stack = used_stack = 0;
1645 } else {
1646 max_used_stack = used_stack = -1;
1647 }
1648
1649 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1650 idx = 0;
1651 level = 0;
1652 opline = NULL;
1653 for (;;p++) {
1654 if (p->op == ZEND_JIT_TRACE_VM) {
1655 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1656 uint8_t val_type = IS_UNKNOWN;
1657 // zend_class_entry *op1_ce = NULL;
1658 zend_class_entry *op2_ce = NULL;
1659
1660 opline = p->opline;
1661
1662 op1_type = orig_op1_type = p->op1_type;
1663 op2_type = orig_op2_type = p->op2_type;
1664 op3_type = p->op3_type;
1665 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1666 op1_type = IS_UNKNOWN;
1667 }
1668 if (op1_type != IS_UNKNOWN) {
1669 op1_type &= ~IS_TRACE_PACKED;
1670 }
1671 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1672 op2_type = IS_UNKNOWN;
1673 }
1674 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1675 op3_type = IS_UNKNOWN;
1676 }
1677
1678 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1679 // op1_ce = (zend_class_entry*)(p+1)->ce;
1680 p++;
1681 }
1682 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1683 op2_ce = (zend_class_entry*)(p+1)->ce;
1684 p++;
1685 }
1686 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1687 val_type = (p+1)->op1_type;
1688 p++;
1689 }
1690
1691 switch (opline->opcode) {
1692 case ZEND_ASSIGN_OP:
1693 if (opline->extended_value == ZEND_POW
1694 || opline->extended_value == ZEND_DIV) {
1695 // TODO: check for division by zero ???
1696 break;
1697 }
1698 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1699 break;
1700 }
1701 ADD_OP1_TRACE_GUARD();
1702 ADD_OP2_TRACE_GUARD();
1703 break;
1704 case ZEND_ASSIGN_DIM_OP:
1705 if (opline->extended_value == ZEND_POW
1706 || opline->extended_value == ZEND_DIV) {
1707 // TODO: check for division by zero ???
1708 break;
1709 }
1710 if (opline->result_type != IS_UNUSED) {
1711 break;
1712 }
1713 if (op3_type != IS_UNKNOWN
1714 && !zend_jit_supported_binary_op(
1715 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1716 break;
1717 }
1718 ZEND_FALLTHROUGH;
1719 case ZEND_ASSIGN_DIM:
1720 if (opline->op1_type == IS_CV) {
1721 if ((opline+1)->op1_type == IS_CV
1722 && (opline+1)->op1.var == opline->op1.var) {
1723 /* skip $a[x] = $a; */
1724 break;
1725 }
1726 ADD_OP1_DATA_TRACE_GUARD();
1727 ADD_OP2_TRACE_GUARD();
1728 ADD_OP1_TRACE_GUARD();
1729 } else if (orig_op1_type != IS_UNKNOWN
1730 && (orig_op1_type & IS_TRACE_INDIRECT)
1731 && opline->result_type == IS_UNUSED) {
1732 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1733 ADD_OP1_DATA_TRACE_GUARD();
1734 }
1735 ADD_OP2_TRACE_GUARD();
1736 }
1737 if (op1_type == IS_ARRAY
1738 && ((opline->op2_type == IS_CONST
1739 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1740 || (opline->op2_type != IS_CONST
1741 && op2_type == IS_LONG))) {
1742
1743 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1744 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1745
1746 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1747 info->type |= MAY_BE_PACKED_GUARD;
1748 info->type &= ~MAY_BE_ARRAY_PACKED;
1749 }
1750 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1751 && val_type != IS_UNKNOWN
1752 && val_type != IS_UNDEF) {
1753 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1754
1755 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1756 info->type |= MAY_BE_PACKED_GUARD;
1757 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1758 }
1759 }
1760 }
1761 break;
1762 case ZEND_ASSIGN_OBJ_OP:
1763 if (opline->extended_value == ZEND_POW
1764 || opline->extended_value == ZEND_DIV) {
1765 // TODO: check for division by zero ???
1766 break;
1767 }
1768 if (opline->result_type != IS_UNUSED) {
1769 break;
1770 }
1771 ZEND_FALLTHROUGH;
1772 case ZEND_ASSIGN_OBJ:
1773 case ZEND_PRE_INC_OBJ:
1774 case ZEND_PRE_DEC_OBJ:
1775 case ZEND_POST_INC_OBJ:
1776 case ZEND_POST_DEC_OBJ:
1777 if (opline->op2_type != IS_CONST
1778 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1779 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1780 break;
1781 }
1782 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1783 if (opline->op1_type == IS_CV
1784 && (opline+1)->op1_type == IS_CV
1785 && (opline+1)->op1.var == opline->op1.var) {
1786 /* skip $a->prop += $a; */
1787 break;
1788 }
1789 ADD_OP1_DATA_TRACE_GUARD();
1790 }
1791 ADD_OP1_TRACE_GUARD();
1792 break;
1793 case ZEND_CONCAT:
1794 case ZEND_FAST_CONCAT:
1795 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1796 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1797 ADD_OP2_TRACE_GUARD();
1798 ADD_OP1_TRACE_GUARD();
1799 }
1800 break;
1801 case ZEND_IS_EQUAL:
1802 case ZEND_IS_NOT_EQUAL:
1803 case ZEND_IS_SMALLER:
1804 case ZEND_IS_SMALLER_OR_EQUAL:
1805 case ZEND_CASE:
1806 case ZEND_IS_IDENTICAL:
1807 case ZEND_IS_NOT_IDENTICAL:
1808 case ZEND_CASE_STRICT:
1809 case ZEND_BW_OR:
1810 case ZEND_BW_AND:
1811 case ZEND_BW_XOR:
1812 case ZEND_SL:
1813 case ZEND_SR:
1814 case ZEND_MOD:
1815 case ZEND_ADD:
1816 case ZEND_SUB:
1817 case ZEND_MUL:
1818 // case ZEND_DIV: // TODO: check for division by zero ???
1819 ADD_OP2_TRACE_GUARD();
1820 ZEND_FALLTHROUGH;
1821 case ZEND_ECHO:
1822 case ZEND_STRLEN:
1823 case ZEND_COUNT:
1824 case ZEND_QM_ASSIGN:
1825 case ZEND_FE_RESET_R:
1826 case ZEND_FE_FETCH_R:
1827 ADD_OP1_TRACE_GUARD();
1828 break;
1829 case ZEND_VERIFY_RETURN_TYPE:
1830 if (opline->op1_type == IS_UNUSED) {
1831 /* Always throws */
1832 break;
1833 }
1834 if (opline->op1_type == IS_CONST) {
1835 /* TODO Different instruction format, has return value */
1836 break;
1837 }
1838 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1839 /* Not worth bothering with */
1840 break;
1841 }
1842 ADD_OP1_TRACE_GUARD();
1843 break;
1844 case ZEND_FETCH_DIM_FUNC_ARG:
1845 if (!frame
1846 || !frame->call
1847 || !frame->call->func
1848 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1849 break;
1850 }
1851 ADD_OP2_TRACE_GUARD();
1852 ADD_OP1_TRACE_GUARD();
1853 break;
1854 case ZEND_PRE_INC:
1855 case ZEND_PRE_DEC:
1856 case ZEND_POST_INC:
1857 case ZEND_POST_DEC:
1858 if (opline->op1_type != IS_CV) {
1859 break;
1860 }
1861 ADD_OP1_TRACE_GUARD();
1862 break;
1863 case ZEND_ASSIGN:
1864 if (opline->op1_type != IS_CV) {
1865 break;
1866 }
1867 ADD_OP2_TRACE_GUARD();
1868 if (op1_type != IS_UNKNOWN
1869 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1870 ADD_OP1_TRACE_GUARD();
1871 }
1872 break;
1873 case ZEND_CAST:
1874 if (opline->extended_value != op1_type) {
1875 break;
1876 }
1877 ADD_OP1_TRACE_GUARD();
1878 break;
1879 case ZEND_JMPZ:
1880 case ZEND_JMPNZ:
1881 case ZEND_JMPZNZ:
1882 case ZEND_JMPZ_EX:
1883 case ZEND_JMPNZ_EX:
1884 case ZEND_BOOL:
1885 case ZEND_BOOL_NOT:
1886 ADD_OP1_TRACE_GUARD();
1887 break;
1888 case ZEND_ISSET_ISEMPTY_CV:
1889 if ((opline->extended_value & ZEND_ISEMPTY)) {
1890 // TODO: support for empty() ???
1891 break;
1892 }
1893 ADD_OP1_TRACE_GUARD();
1894 break;
1895 case ZEND_IN_ARRAY:
1896 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
1897 break;
1898 }
1899 ADD_OP1_TRACE_GUARD();
1900 break;
1901 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
1902 if ((opline->extended_value & ZEND_ISEMPTY)) {
1903 // TODO: support for empty() ???
1904 break;
1905 }
1906 ZEND_FALLTHROUGH;
1907 case ZEND_FETCH_DIM_R:
1908 case ZEND_FETCH_DIM_IS:
1909 case ZEND_FETCH_LIST_R:
1910 ADD_OP1_TRACE_GUARD();
1911 ADD_OP2_TRACE_GUARD();
1912
1913 if (op1_type == IS_ARRAY
1914 && opline->op1_type != IS_CONST
1915 && ((opline->op2_type == IS_CONST
1916 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1917 || (opline->op2_type != IS_CONST
1918 && op2_type == IS_LONG))) {
1919
1920 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1921
1922 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1923 info->type |= MAY_BE_PACKED_GUARD;
1924 if (orig_op1_type & IS_TRACE_PACKED) {
1925 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1926 } else {
1927 info->type &= ~MAY_BE_ARRAY_PACKED;
1928 }
1929 }
1930 }
1931 break;
1932 case ZEND_FETCH_DIM_W:
1933 case ZEND_FETCH_DIM_RW:
1934 // case ZEND_FETCH_DIM_UNSET:
1935 case ZEND_FETCH_LIST_W:
1936 if (opline->op1_type != IS_CV
1937 && (orig_op1_type == IS_UNKNOWN
1938 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
1939 break;
1940 }
1941 ADD_OP1_TRACE_GUARD();
1942 ADD_OP2_TRACE_GUARD();
1943 if (op1_type == IS_ARRAY
1944 && !(orig_op1_type & IS_TRACE_PACKED)
1945 && ((opline->op2_type == IS_CONST
1946 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1947 || (opline->op2_type != IS_CONST
1948 && op2_type == IS_LONG))) {
1949
1950 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1951
1952 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1953 info->type |= MAY_BE_PACKED_GUARD;
1954 info->type &= ~MAY_BE_ARRAY_PACKED;
1955 }
1956 }
1957 break;
1958 case ZEND_SEND_VAL_EX:
1959 case ZEND_SEND_VAR_EX:
1960 case ZEND_SEND_VAR_NO_REF_EX:
1961 if (opline->op2_type == IS_CONST) {
1962 /* Named parameters not supported in JIT */
1963 break;
1964 }
1965 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
1966 goto propagate_arg;
1967 }
1968 ZEND_FALLTHROUGH;
1969 case ZEND_SEND_VAL:
1970 case ZEND_SEND_VAR:
1971 case ZEND_SEND_VAR_NO_REF:
1972 case ZEND_SEND_FUNC_ARG:
1973 if (opline->op2_type == IS_CONST) {
1974 /* Named parameters not supported in JIT */
1975 break;
1976 }
1977 ADD_OP1_TRACE_GUARD();
1978 propagate_arg:
1979 /* Propagate argument type */
1980 if (frame->call
1981 && frame->call->func
1982 && frame->call->func->type == ZEND_USER_FUNCTION
1983 && opline->op2.num <= frame->call->func->op_array.num_args) {
1984 uint32_t info;
1985
1986 if (opline->op1_type == IS_CONST) {
1987 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
1988 } else {
1989 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
1990 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
1991 }
1992 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
1993 zend_arg_info *arg_info;
1994
1995 ZEND_ASSERT(frame->call->func->op_array.arg_info);
1996 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
1997 if (ZEND_TYPE_IS_SET(arg_info->type)) {
1998 zend_class_entry *ce;
1999 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2000 info &= tmp;
2001 if (!info) {
2002 break;
2003 }
2004 }
2005 }
2006 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2007 info |= MAY_BE_RCN;
2008 }
2009 if (info & MAY_BE_UNDEF) {
2010 info |= MAY_BE_NULL;
2011 info &= ~MAY_BE_UNDEF;
2012 }
2013 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2014 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2015 }
2016 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2017 }
2018 break;
2019 case ZEND_RETURN:
2020 ADD_OP1_TRACE_GUARD();
2021 /* Propagate return value types */
2022 if (opline->op1_type == IS_UNUSED) {
2023 return_value_info.type = MAY_BE_NULL;
2024 } else if (opline->op1_type == IS_CONST) {
2025 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2026 } else {
2027 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2028 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2029 if (return_value_info.type & MAY_BE_UNDEF) {
2030 return_value_info.type &= ~MAY_BE_UNDEF;
2031 return_value_info.type |= MAY_BE_NULL;
2032 }
2033 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2034 /* CVs are going to be destructed and the reference-counter
2035 of return value may be decremented to 1 */
2036 return_value_info.type |= MAY_BE_RC1;
2037 }
2038 return_value_info.type &= ~MAY_BE_GUARD;
2039 }
2040 break;
2041 case ZEND_CHECK_FUNC_ARG:
2042 if (!frame
2043 || !frame->call
2044 || !frame->call->func) {
2045 break;
2046 }
2047 if (opline->op2_type == IS_CONST
2048 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2049 /* Named parameters not supported in JIT */
2050 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2051 break;
2052 }
2053 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2054 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2055 } else {
2056 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2057 }
2058 break;
2059 case ZEND_FETCH_OBJ_FUNC_ARG:
2060 if (!frame
2061 || !frame->call
2062 || !frame->call->func
2063 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2064 break;
2065 }
2066 ZEND_FALLTHROUGH;
2067 case ZEND_FETCH_OBJ_R:
2068 case ZEND_FETCH_OBJ_IS:
2069 case ZEND_FETCH_OBJ_W:
2070 if (opline->op2_type != IS_CONST
2071 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2072 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2073 break;
2074 }
2075 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2076 ADD_OP1_TRACE_GUARD();
2077 }
2078 break;
2079 case ZEND_INIT_METHOD_CALL:
2080 if (opline->op2_type != IS_CONST
2081 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2082 break;
2083 }
2084 ADD_OP1_TRACE_GUARD();
2085 break;
2086 case ZEND_INIT_DYNAMIC_CALL:
2087 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2088 ADD_OP2_TRACE_GUARD();
2089 }
2090 break;
2091 case ZEND_SEND_ARRAY:
2092 case ZEND_SEND_UNPACK:
2093 case ZEND_CHECK_UNDEF_ARGS:
2094 case ZEND_INCLUDE_OR_EVAL:
2095 max_used_stack = used_stack = -1;
2096 break;
2097 case ZEND_TYPE_CHECK:
2098 if (opline->extended_value == MAY_BE_RESOURCE) {
2099 // TODO: support for is_resource() ???
2100 break;
2101 }
2102 if (op1_type != IS_UNKNOWN
2103 && (opline->extended_value == (1 << op1_type)
2104 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2105 /* add guards only for exact checks, to avoid code duplication */
2106 ADD_OP1_TRACE_GUARD();
2107 }
2108 break;
2109 case ZEND_ROPE_INIT:
2110 case ZEND_ROPE_ADD:
2111 case ZEND_ROPE_END:
2112 if (op2_type == IS_STRING) {
2113 ADD_OP2_TRACE_GUARD();
2114 }
2115 break;
2116 default:
2117 break;
2118 }
2119 len = zend_jit_trace_op_len(opline);
2120 if (ssa->var_info) {
2121 /* Add statically inferred ranges */
2122 if (ssa_ops[idx].op1_def >= 0) {
2123 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2124 }
2125 if (ssa_ops[idx].op2_def >= 0) {
2126 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2127 }
2128 if (ssa_ops[idx].result_def >= 0) {
2129 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2130 }
2131 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2132 if (ssa_ops[idx+1].op1_def >= 0) {
2133 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2134 }
2135 if (ssa_ops[idx+1].op2_def >= 0) {
2136 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2137 }
2138 if (ssa_ops[idx+1].result_def >= 0) {
2139 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2140 }
2141 }
2142 } else {
2143 if (ssa_ops[idx].op1_def >= 0) {
2144 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2145 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2146 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2147 }
2148 }
2149 if (ssa_ops[idx].op2_def >= 0) {
2150 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2151 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2152 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2153 }
2154 }
2155 if (ssa_ops[idx].result_def >= 0) {
2156 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2157 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2158 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2159 }
2160 }
2161 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2162 if (ssa_ops[idx+1].op1_def >= 0) {
2163 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2164 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2165 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2166 }
2167 }
2168 if (ssa_ops[idx+1].op2_def >= 0) {
2169 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2170 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2171 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2172 }
2173 }
2174 if (ssa_ops[idx+1].result_def >= 0) {
2175 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2176 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2177 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2178 }
2179 }
2180 }
2181 }
2182 if (opline->opcode == ZEND_RECV_INIT
2183 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2184 /* RECV_INIT always copy the constant */
2185 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2186 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2187 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2188 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2189 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2190 }
2191 } else {
2192 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2193 // TODO:
2194 assert(0);
2195 }
2196 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2197 && ssa_ops[idx].op1_def > 0
2198 && op1_type == IS_ARRAY
2199 && (orig_op1_type & IS_TRACE_PACKED)
2200 && val_type != IS_UNKNOWN
2201 && val_type != IS_UNDEF
2202 && ((opline->op2_type == IS_CONST
2203 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2204 || (opline->op2_type != IS_CONST
2205 && op2_type == IS_LONG))) {
2206 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2207
2208 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2209 }
2210 }
2211 if (ssa->var_info) {
2212 /* Add statically inferred restrictions */
2213 if (ssa_ops[idx].op1_def >= 0) {
2214 if (opline->opcode == ZEND_SEND_VAR_EX
2215 && frame
2216 && frame->call
2217 && frame->call->func
2218 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2219 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2220 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2221 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2222 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2223 }
2224 } else {
2225 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2226 }
2227 }
2228 if (ssa_ops[idx].op2_def >= 0) {
2229 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2230 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2231 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2232 }
2233 }
2234 if (ssa_ops[idx].result_def >= 0) {
2235 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2236 }
2237 }
2238 idx++;
2239 while (len > 1) {
2240 opline++;
2241 if (opline->opcode != ZEND_OP_DATA) {
2242 if (ssa->var_info) {
2243 /* Add statically inferred ranges */
2244 if (ssa_ops[idx].op1_def >= 0) {
2245 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2246 }
2247 if (ssa_ops[idx].op2_def >= 0) {
2248 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2249 }
2250 if (ssa_ops[idx].result_def >= 0) {
2251 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2252 }
2253 } else {
2254 if (ssa_ops[idx].op1_def >= 0) {
2255 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2256 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2257 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2258 }
2259 }
2260 if (ssa_ops[idx].op2_def >= 0) {
2261 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2262 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2263 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2264 }
2265 }
2266 if (ssa_ops[idx].result_def >= 0) {
2267 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2268 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2269 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2270 }
2271 }
2272 }
2273 if (opline->opcode == ZEND_RECV_INIT
2274 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2275 /* RECV_INIT always copy the constant */
2276 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2277 } else {
2278 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2279 // TODO:
2280 assert(0);
2281 }
2282 }
2283 }
2284 if (ssa->var_info) {
2285 /* Add statically inferred restrictions */
2286 if (ssa_ops[idx].op1_def >= 0) {
2287 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2288 }
2289 if (ssa_ops[idx].op2_def >= 0) {
2290 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2291 }
2292 if (ssa_ops[idx].result_def >= 0) {
2293 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2294 }
2295 }
2296 idx++;
2297 len--;
2298 }
2299
2300 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2301 op_array = p->op_array;
2302 jit_extension =
2303 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2304 ssa = &jit_extension->func_info.ssa;
2305
2306 call = frame->call;
2307 if (!call) {
2308 /* Trace missed INIT_FCALL opcode */
2309 call = top;
2310 TRACE_FRAME_INIT(call, op_array, 0, 0);
2311 call->used_stack = 0;
2312 top = zend_jit_trace_call_frame(top, op_array);
2313 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2314 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
2315 }
2316 } else {
2317 ZEND_ASSERT(&call->func->op_array == op_array);
2318 }
2319 frame->call = call->prev;
2320 call->prev = frame;
2321 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2322 frame = call;
2323
2324 level++;
2325 i = 0;
2326 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2327 while (i < op_array->last_var) {
2328 ssa_vars[v].var = i;
2329 if (i < op_array->num_args) {
2330 if (ssa->var_info
2331 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2332 /* pass */
2333 } else {
2334 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2335 if (op_array->arg_info) {
2336 zend_arg_info *arg_info = &op_array->arg_info[i];
2337 zend_class_entry *ce;
2338 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2339
2340 if (ZEND_ARG_SEND_MODE(arg_info)) {
2341 tmp |= MAY_BE_REF;
2342 }
2343 ssa_var_info[v].type = tmp;
2344 ssa_var_info[v].ce = ce;
2345 ssa_var_info[v].is_instanceof = 1;
2346 } else {
2347 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2348 }
2349 }
2350 } else {
2351 if (ssa->vars) {
2352 ssa_vars[v].no_val = ssa->vars[i].no_val;
2353 ssa_vars[v].alias = ssa->vars[i].alias;
2354 } else {
2355 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2356 }
2357 if (ssa_vars[v].alias == NO_ALIAS) {
2358 ssa_var_info[v].type = MAY_BE_UNDEF;
2359 } else {
2360 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2361 }
2362 }
2363 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2364 && i < op_array->num_args) {
2365 /* Propagate argument type */
2366 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2367 }
2368 i++;
2369 v++;
2370 }
2371 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2372 op_array = p->op_array;
2373 jit_extension =
2374 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2375 ssa = &jit_extension->func_info.ssa;
2376 if (level == 0) {
2377 i = 0;
2378 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2379 while (i < op_array->last_var) {
2380 ssa_vars[v].var = i;
2381 if (!ssa->var_info
2382 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2383 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2384 }
2385 i++;
2386 v++;
2387 }
2388 while (i < op_array->last_var + op_array->T) {
2389 ssa_vars[v].var = i;
2390 if (!ssa->var_info
2391 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2392 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2393 }
2394 i++;
2395 v++;
2396 }
2397 if (return_value_info.type != 0) {
2398 zend_jit_trace_rec *q = p + 1;
2399 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2400 q++;
2401 }
2402 if (q->op == ZEND_JIT_TRACE_VM
2403 || (q->op == ZEND_JIT_TRACE_END
2404 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2405 const zend_op *opline = q->opline - 1;
2406 if (opline->result_type != IS_UNUSED) {
2407 ssa_var_info[
2408 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2409 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2410 }
2411 }
2412 memset(&return_value_info, 0, sizeof(return_value_info));
2413 }
2414 } else {
2415 level--;
2416 if (return_value_info.type != 0) {
2417 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2418 const zend_op *opline = (p+1)->opline - 1;
2419 if (opline->result_type != IS_UNUSED) {
2420 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2421 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2422 }
2423 }
2424 }
2425 memset(&return_value_info, 0, sizeof(return_value_info));
2426 }
2427 }
2428
2429 top = frame;
2430 if (frame->prev) {
2431 if (used_stack > 0) {
2432 used_stack -= frame->used_stack;
2433 }
2434 frame = frame->prev;
2435 ZEND_ASSERT(&frame->func->op_array == op_array);
2436 } else {
2437 max_used_stack = used_stack = -1;
2438 frame = zend_jit_trace_ret_frame(frame, op_array);
2439 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2440 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2441 frame->used_stack = 0;
2442 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2443 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
2444 }
2445 }
2446
2447 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2448 call = top;
2449 TRACE_FRAME_INIT(call, p->func, 0, 0);
2450 call->prev = frame->call;
2451 call->used_stack = 0;
2452 frame->call = call;
2453 top = zend_jit_trace_call_frame(top, p->op_array);
2454 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2455 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2456 SET_STACK_INFO(call->stack, i, -1);
2457 }
2458 }
2459 if (used_stack >= 0
2460 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2461 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2462 max_used_stack = used_stack = -1;
2463 } else {
2464 const zend_op *opline = (p-1)->opline;
2465
2466 switch (opline->opcode) {
2467 case ZEND_INIT_FCALL:
2468 case ZEND_INIT_FCALL_BY_NAME:
2469 case ZEND_INIT_NS_FCALL_BY_NAME:
2470 case ZEND_INIT_METHOD_CALL:
2471 case ZEND_INIT_DYNAMIC_CALL:
2472 //case ZEND_INIT_STATIC_METHOD_CALL:
2473 //case ZEND_INIT_USER_CALL:
2474 //case ZEND_NEW:
2475 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2476 used_stack += frame->used_stack;
2477 if (used_stack > max_used_stack) {
2478 max_used_stack = used_stack;
2479 }
2480 break;
2481 default:
2482 max_used_stack = used_stack = -1;
2483 }
2484 }
2485 }
2486 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2487 call = frame->call;
2488 if (call) {
2489 top = call;
2490 frame->call = call->prev;
2491 }
2492
2493 if (idx > 0
2494 && ssa_ops[idx-1].result_def >= 0
2495 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2496 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2497 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2498 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2499 opline->opcode == ZEND_DO_FCALL ||
2500 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2501
2502 if (opline->result_type != IS_UNDEF) {
2503 zend_class_entry *ce;
2504 const zend_function *func = p->func;
2505 zend_arg_info *ret_info = func->common.arg_info - 1;
2506 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2507
2508 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2509 }
2510 }
2511 } else if (p->op == ZEND_JIT_TRACE_END) {
2512 break;
2513 }
2514 }
2515
2516 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2517
2518 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2519 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2520 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2521 /* Propagate guards through Phi sources */
2522 zend_ssa_phi *phi = tssa->blocks[1].phis;
2523
2524 op_array = trace_buffer->op_array;
2525 jit_extension =
2526 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2527 ssa = &jit_extension->func_info.ssa;
2528
2529 while (phi) {
2530 uint32_t t = ssa_var_info[phi->ssa_var].type;
2531
2532 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2533 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2534 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2535
2536 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2537 if (!((t0 | t1) & MAY_BE_GUARD)) {
2538 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2539 }
2540 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2541 if (!(t1 & MAY_BE_GUARD)
2542 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2543 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2544 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2545 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2546 MAY_BE_GUARD;
2547 if (!(t0 & MAY_BE_ARRAY)) {
2548 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2549 }
2550 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2551 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2552 }
2553 ssa_var_info[phi->sources[0]].type = t0;
2554 ssa_var_info[phi->sources[0]].type = t0;
2555 }
2556 } else {
2557 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2558 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2559 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2560 MAY_BE_GUARD;
2561 if (!(t0 & MAY_BE_ARRAY)) {
2562 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2563 }
2564 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2565 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2566 }
2567 ssa_var_info[phi->sources[0]].type = t0;
2568 }
2569 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2570 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2571 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2572 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2573 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2574 MAY_BE_GUARD;
2575 if (!(t1 & MAY_BE_ARRAY)) {
2576 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2577 }
2578 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2579 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2580 }
2581 ssa_var_info[phi->sources[1]].type = t1;
2582 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2583 }
2584 }
2585 }
2586 t = ssa_var_info[phi->ssa_var].type;
2587 }
2588
2589 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2590 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2591 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2592
2593 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2594 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2595 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2596 }
2597 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2598 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2599 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2600 ssa_var_info[phi->sources[0]].type =
2601 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2602 }
2603 }
2604 }
2605 phi = phi->next;
2606 }
2607 }
2608
2609 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2610 if (parent_trace) {
2611 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2612 ZEND_JIT_TRACE_NUM,
2613 parent_trace,
2614 exit_num,
2615 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2616 trace_buffer->op_array->scope ? "::" : "",
2617 trace_buffer->op_array->function_name ?
2618 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2619 ZSTR_VAL(trace_buffer->op_array->filename),
2620 trace_buffer[1].opline->lineno);
2621 } else {
2622 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2623 ZEND_JIT_TRACE_NUM,
2624 zend_jit_trace_star_desc(trace_buffer->start),
2625 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2626 trace_buffer->op_array->scope ? "::" : "",
2627 trace_buffer->op_array->function_name ?
2628 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2629 ZSTR_VAL(trace_buffer->op_array->filename),
2630 trace_buffer[1].opline->lineno);
2631 }
2632 zend_jit_dump_trace(trace_buffer, tssa);
2633 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2634 uint32_t idx = trace_buffer[1].last;
2635 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2636 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2637 ZEND_JIT_TRACE_NUM,
2638 link_to);
2639 } else {
2640 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2641 ZEND_JIT_TRACE_NUM,
2642 zend_jit_trace_stop_description[trace_buffer->stop]);
2643 }
2644 }
2645
2646 return tssa;
2647 }
2648
zend_jit_close_var(zend_jit_trace_stack * stack,uint32_t n,int * start,int * end,uint8_t * flags,int line)2649 static void zend_jit_close_var(zend_jit_trace_stack *stack, uint32_t n, int *start, int *end, uint8_t *flags, int line)
2650 {
2651 int32_t var = STACK_VAR(stack, n);
2652
2653 if (var >= 0 && start[var] >= 0 && !(flags[var] & ZREG_LAST_USE)) {
2654 // TODO: shrink interval to last side exit ????
2655 end[var] = line;
2656 }
2657 }
2658
zend_jit_trace_use_var(int line,int var,int def,int use_chain,int * start,int * end,uint8_t * flags,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2659 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, int *start, int *end, uint8_t *flags, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2660 {
2661 ZEND_ASSERT(start[var] >= 0);
2662 ZEND_ASSERT(!(flags[var] & ZREG_LAST_USE));
2663 end[var] = line;
2664 if (def >= 0) {
2665 flags[var] |= ZREG_LAST_USE;
2666 } else if (use_chain < 0 && (flags[var] & (ZREG_LOAD|ZREG_STORE))) {
2667 flags[var] |= ZREG_LAST_USE;
2668 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2669 /* pass */
2670 } else if (op_array_ssa->vars) {
2671 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2672
2673 if (ssa->ops[line].op1_use == var) {
2674 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2675 flags[var] |= ZREG_LAST_USE;
2676 }
2677 } else if (ssa->ops[line].op2_use == var) {
2678 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2679 flags[var] |= ZREG_LAST_USE;
2680 }
2681 } else if (ssa->ops[line].result_use == var) {
2682 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2683 flags[var] |= ZREG_LAST_USE;
2684 }
2685 }
2686 }
2687 }
2688
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2689 static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2690 {
2691 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2692 zend_jit_trace_rec *p;
2693 const zend_op_array *op_array;
2694 zend_jit_op_array_trace_extension *jit_extension;
2695 const zend_ssa *op_array_ssa;
2696 const zend_ssa_op *ssa_op;
2697 int i, j, idx, count, level;
2698 int last_idx = -1;
2699 int *start, *end;
2700 uint8_t *flags;
2701 const zend_op_array **vars_op_array;
2702 zend_lifetime_interval **intervals, *list, *ival;
2703 void *checkpoint;
2704 zend_jit_trace_stack_frame *frame;
2705 zend_jit_trace_stack *stack;
2706 uint32_t parent_vars_count = parent_trace ?
2707 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2708 zend_jit_trace_stack *parent_stack = parent_trace ?
2709 zend_jit_traces[parent_trace].stack_map +
2710 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2711 ALLOCA_FLAG(use_heap);
2712
2713 ZEND_ASSERT(ssa->var_info != NULL);
2714
2715 start = do_alloca(sizeof(int) * ssa->vars_count * 2 +
2716 ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count),
2717 use_heap);
2718 if (!start) {
2719 return NULL;
2720 }
2721 end = start + ssa->vars_count;
2722 flags = (uint8_t*)(end + ssa->vars_count);
2723 checkpoint = zend_arena_checkpoint(CG(arena));
2724 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2725
2726 memset(start, -1, sizeof(int) * ssa->vars_count * 2);
2727 memset(flags, 0, sizeof(uint8_t) * ssa->vars_count);
2728 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2729
2730 op_array = trace_buffer->op_array;
2731 jit_extension =
2732 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2733 op_array_ssa = &jit_extension->func_info.ssa;
2734 frame = JIT_G(current_frame);
2735 frame->prev = NULL;
2736 frame->func = (const zend_function*)op_array;
2737 stack = frame->stack;
2738
2739 count = 0;
2740
2741 i = 0;
2742 j = op_array->last_var;
2743 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2744 j += op_array->T;
2745 }
2746 while (i < j) {
2747 SET_STACK_VAR(stack, i, i);
2748 vars_op_array[i] = op_array;
2749 /* We don't start intervals for variables used in Phi */
2750 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2751 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2752 && ssa->vars[i].alias == NO_ALIAS
2753 && zend_jit_var_supports_reg(ssa, i)) {
2754 start[i] = 0;
2755 if (i < parent_vars_count
2756 && STACK_REG(parent_stack, i) != ZREG_NONE
2757 && STACK_REG(parent_stack, i) < ZREG_NUM) {
2758 /* We will try to reuse register from parent trace */
2759 flags[i] = STACK_FLAGS(parent_stack, i);
2760 count += 2;
2761 } else {
2762 flags[i] = ZREG_LOAD;
2763 count++;
2764 }
2765 }
2766 i++;
2767 }
2768
2769 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2770 j = op_array->last_var + op_array->T;
2771 while (i < j) {
2772 SET_STACK_VAR(stack, i, -1);
2773 i++;
2774 }
2775 }
2776
2777 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2778 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2779 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2780 zend_ssa_phi *phi = ssa->blocks[1].phis;
2781
2782 while (phi) {
2783 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2784 vars_op_array[phi->ssa_var] = op_array;
2785 if (ssa->vars[phi->ssa_var].use_chain >= 0
2786 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2787 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2788 start[phi->ssa_var] = 0;
2789 count++;
2790 }
2791 phi = phi->next;
2792 }
2793 }
2794
2795 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2796 level = 0;
2797 ssa_op = ssa->ops;
2798 idx = 0;
2799 for (;;p++) {
2800 if (p->op == ZEND_JIT_TRACE_VM) {
2801 const zend_op *opline = p->opline;
2802 int len;
2803 bool support_opline;
2804
2805 support_opline =
2806 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2807
2808 if (support_opline
2809 && opline->opcode == ZEND_ASSIGN
2810 && opline->op1_type == IS_CV
2811 && ssa_op->op1_def >= 0
2812 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2813 /* avoid register allocation in case of possibility of indirect modification*/
2814 support_opline = 0;
2815 }
2816
2817 if (ssa_op->op1_use >= 0
2818 && start[ssa_op->op1_use] >= 0
2819 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2820 if (support_opline) {
2821 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2822 if (opline->op1_type != IS_CV) {
2823 if (opline->opcode == ZEND_CASE
2824 || opline->opcode == ZEND_CASE_STRICT
2825 || opline->opcode == ZEND_SWITCH_LONG
2826 || opline->opcode == ZEND_MATCH
2827 || opline->opcode == ZEND_FETCH_LIST_R
2828 || opline->opcode == ZEND_COPY_TMP
2829 || opline->opcode == ZEND_SWITCH_STRING
2830 || opline->opcode == ZEND_FE_FETCH_R
2831 || opline->opcode == ZEND_FE_FETCH_RW
2832 || opline->opcode == ZEND_FETCH_LIST_W
2833 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2834 || opline->opcode == ZEND_BIND_LEXICAL
2835 || opline->opcode == ZEND_ROPE_ADD) {
2836 /* The value is kept alive and may be used outside of the trace */
2837 flags[ssa_op->op1_use] |= ZREG_STORE;
2838 } else {
2839 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2840 }
2841 }
2842 } else {
2843 start[ssa_op->op1_use] = -1;
2844 end[ssa_op->op1_use] = -1;
2845 count--;
2846 }
2847 }
2848 if (ssa_op->op2_use >= 0
2849 && ssa_op->op2_use != ssa_op->op1_use
2850 && start[ssa_op->op2_use] >= 0
2851 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2852 if (support_opline) {
2853 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2854 if (opline->op2_type != IS_CV) {
2855 flags[ssa_op->op2_use] |= ZREG_LAST_USE;
2856 }
2857 } else {
2858 start[ssa_op->op2_use] = -1;
2859 end[ssa_op->op2_use] = -1;
2860 count--;
2861 }
2862 }
2863 if (ssa_op->result_use >= 0
2864 && ssa_op->result_use != ssa_op->op1_use
2865 && ssa_op->result_use != ssa_op->op2_use
2866 && start[ssa_op->result_use] >= 0
2867 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2868 if (support_opline) {
2869 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2870 } else {
2871 start[ssa_op->result_use] = -1;
2872 end[ssa_op->result_use] = -1;
2873 count--;
2874 }
2875 }
2876
2877 if (ssa_op->op1_def >= 0) {
2878 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2879 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2880 }
2881 if (ssa_op->op2_def >= 0) {
2882 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op2.var), start, end, flags, idx);
2883 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
2884 }
2885 if (ssa_op->result_def >= 0) {
2886 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2887 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2888 }
2889
2890 if (support_opline) {
2891 if (ssa_op->result_def >= 0
2892 && (ssa->vars[ssa_op->result_def].use_chain >= 0
2893 || ssa->vars[ssa_op->result_def].phi_use_chain)
2894 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
2895 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
2896 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
2897 || opline->opcode == ZEND_PRE_INC
2898 || opline->opcode == ZEND_PRE_DEC
2899 || opline->opcode == ZEND_POST_INC
2900 || opline->opcode == ZEND_POST_DEC
2901 || opline->opcode == ZEND_ADD
2902 || opline->opcode == ZEND_SUB
2903 || opline->opcode == ZEND_MUL
2904 || opline->opcode == ZEND_FETCH_DIM_R
2905 || opline->opcode == ZEND_FETCH_CONSTANT) {
2906 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
2907 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
2908 start[ssa_op->result_def] = idx;
2909 vars_op_array[ssa_op->result_def] = op_array;
2910 count++;
2911 }
2912 }
2913 }
2914 if (ssa_op->op1_def >= 0
2915 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2916 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2917 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2918 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2919 start[ssa_op->op1_def] = idx;
2920 vars_op_array[ssa_op->op1_def] = op_array;
2921 count++;
2922 }
2923 if (ssa_op->op2_def >= 0
2924 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
2925 || ssa->vars[ssa_op->op2_def].phi_use_chain)
2926 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
2927 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)) {
2928 start[ssa_op->op2_def] = idx;
2929 vars_op_array[ssa_op->op2_def] = op_array;
2930 count++;
2931 }
2932 }
2933
2934 len = zend_jit_trace_op_len(opline);
2935 switch (opline->opcode) {
2936 case ZEND_ASSIGN_DIM:
2937 case ZEND_ASSIGN_OBJ:
2938 case ZEND_ASSIGN_STATIC_PROP:
2939 case ZEND_ASSIGN_DIM_OP:
2940 case ZEND_ASSIGN_OBJ_OP:
2941 case ZEND_ASSIGN_STATIC_PROP_OP:
2942 case ZEND_ASSIGN_OBJ_REF:
2943 case ZEND_ASSIGN_STATIC_PROP_REF:
2944 /* OP_DATA */
2945 ssa_op++;
2946 opline++;
2947 if (ssa_op->op1_use >= 0
2948 && start[ssa_op->op1_use] >= 0
2949 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2950 if (support_opline) {
2951 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2952 if (opline->op1_type != IS_CV) {
2953 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2954 }
2955 } else {
2956 start[ssa_op->op1_use] = -1;
2957 end[ssa_op->op1_use] = -1;
2958 count--;
2959 }
2960 }
2961 if (ssa_op->op1_def >= 0) {
2962 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2963 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2964 if (support_opline
2965 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2966 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2967 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2968 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2969 start[ssa_op->op1_def] = idx;
2970 vars_op_array[ssa_op->op1_def] = op_array;
2971 count++;
2972 }
2973 }
2974 ssa_op++;
2975 opline++;
2976 idx+=2;
2977 break;
2978 case ZEND_RECV_INIT:
2979 ssa_op++;
2980 opline++;
2981 idx++;
2982 while (opline->opcode == ZEND_RECV_INIT) {
2983 /* RECV_INIT doesn't support registers */
2984 if (ssa_op->result_def >= 0) {
2985 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2986 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2987 }
2988 ssa_op++;
2989 opline++;
2990 idx++;
2991 }
2992 break;
2993 case ZEND_BIND_GLOBAL:
2994 ssa_op++;
2995 opline++;
2996 idx++;
2997 while (opline->opcode == ZEND_BIND_GLOBAL) {
2998 /* BIND_GLOBAL doesn't support registers */
2999 if (ssa_op->op1_def >= 0) {
3000 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
3001 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3002 }
3003 ssa_op++;
3004 opline++;
3005 idx++;
3006 }
3007 break;
3008 default:
3009 ssa_op += len;
3010 idx += len;
3011 break;
3012 }
3013 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3014 /* New call frames */
3015 zend_jit_trace_stack_frame *prev_frame = frame;
3016
3017 frame = zend_jit_trace_call_frame(frame, op_array);
3018 frame->prev = prev_frame;
3019 frame->func = (const zend_function*)p->op_array;
3020 stack = frame->stack;
3021 op_array = p->op_array;
3022 jit_extension =
3023 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3024 op_array_ssa = &jit_extension->func_info.ssa;
3025 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3026 for (i = 0; i < op_array->last_var; i++) {
3027 SET_STACK_VAR(stack, i, j);
3028 vars_op_array[j] = op_array;
3029 if (ssa->vars[j].use_chain >= 0
3030 && ssa->vars[j].alias == NO_ALIAS
3031 && zend_jit_var_supports_reg(ssa, j)) {
3032 start[j] = idx;
3033 flags[j] = ZREG_LOAD;
3034 count++;
3035 }
3036 j++;
3037 }
3038 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3039 SET_STACK_VAR(stack, i, -1);
3040 }
3041 level++;
3042 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3043 /* Close exiting call frames */
3044 for (i = 0; i < op_array->last_var; i++) {
3045 zend_jit_close_var(stack, i, start, end, flags, idx-1);
3046 }
3047 op_array = p->op_array;
3048 jit_extension =
3049 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3050 op_array_ssa = &jit_extension->func_info.ssa;
3051 frame = zend_jit_trace_ret_frame(frame, op_array);
3052 stack = frame->stack;
3053 if (level == 0) {
3054 /* New return frames */
3055 frame->prev = NULL;
3056 frame->func = (const zend_function*)op_array;
3057 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3058 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3059 SET_STACK_VAR(stack, i, j);
3060 vars_op_array[j] = op_array;
3061 if (ssa->vars[j].use_chain >= 0
3062 && ssa->vars[j].alias == NO_ALIAS
3063 && zend_jit_var_supports_reg(ssa, j)) {
3064 start[j] = idx;
3065 flags[j] = ZREG_LOAD;
3066 count++;
3067 }
3068 j++;
3069 }
3070 } else {
3071 level--;
3072 }
3073 } else if (p->op == ZEND_JIT_TRACE_END) {
3074 break;
3075 }
3076 }
3077
3078 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3079 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3080 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3081 zend_ssa_phi *phi = ssa->blocks[1].phis;
3082
3083 while (phi) {
3084 i = phi->sources[1];
3085 if (start[i] >= 0 && !ssa->vars[phi->ssa_var].no_val) {
3086 end[i] = idx;
3087 flags[i] &= ~ZREG_LAST_USE;
3088 }
3089 phi = phi->next;
3090 }
3091
3092 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3093 for (i = 0; i < op_array->last_var; i++) {
3094 if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) {
3095 end[i] = idx;
3096 flags[i] &= ~ZREG_LAST_USE;
3097 } else {
3098 zend_jit_close_var(stack, i, start, end, flags, idx);
3099 }
3100 }
3101 }
3102 } else {
3103 last_idx = idx;
3104 for (i = 0; i < op_array->last_var; i++) {
3105 zend_jit_close_var(stack, i, start, end, flags, idx);
3106 }
3107 while (frame->prev) {
3108 frame = frame->prev;
3109 op_array = &frame->func->op_array;
3110 stack = frame->stack;
3111 for (i = 0; i < op_array->last_var; i++) {
3112 zend_jit_close_var(stack, i, start, end, flags, idx);
3113 }
3114 }
3115 }
3116
3117 if (!count) {
3118 free_alloca(start, use_heap);
3119 zend_arena_release(&CG(arena), checkpoint);
3120 return NULL;
3121 }
3122
3123 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval));
3124 memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count);
3125 list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count);
3126 j = 0;
3127 for (i = 0; i < ssa->vars_count; i++) {
3128 if (start[i] >= 0 && end[i] >= 0) {
3129 ZEND_ASSERT(j < count);
3130 if ((flags[i] & ZREG_LOAD) &&
3131 (flags[i] & ZREG_LAST_USE) &&
3132 end[i] == ssa->vars[i].use_chain) {
3133 /* skip life range with single use */
3134 continue;
3135 }
3136 intervals[i] = &list[j];
3137 list[j].ssa_var = i;
3138 list[j].reg = ZREG_NONE;
3139 list[j].flags = flags[i];
3140 list[j].range.start = start[i];
3141 list[j].range.end = end[i];
3142 list[j].range.next = NULL;
3143 list[j].hint = NULL;
3144 list[j].used_as_hint = NULL;
3145 list[j].list_next = NULL;
3146 j++;
3147 }
3148 }
3149 count = j;
3150 free_alloca(start, use_heap);
3151 start = end = NULL;
3152
3153 if (!count) {
3154 zend_arena_release(&CG(arena), checkpoint);
3155 return NULL;
3156 }
3157
3158 /* Add hints */
3159 if (parent_vars_count) {
3160 i = trace_buffer->op_array->last_var;
3161 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3162 i += trace_buffer->op_array->T;
3163 }
3164 if ((uint32_t)i > parent_vars_count) {
3165 i = parent_vars_count;
3166 }
3167 while (i > 0) {
3168 i--;
3169 if (intervals[i]
3170 && STACK_REG(parent_stack, i) != ZREG_NONE
3171 && STACK_REG(parent_stack, i) < ZREG_NUM) {
3172 list[j].ssa_var = - 1;
3173 list[j].reg = STACK_REG(parent_stack, i);
3174 list[j].flags = 0;
3175 list[j].range.start = -1;
3176 list[j].range.end = -1;
3177 list[j].range.next = NULL;
3178 list[j].hint = NULL;
3179 list[j].used_as_hint = NULL;
3180 list[j].list_next = NULL;
3181 intervals[i]->hint = &list[j];
3182 j++;
3183 }
3184 }
3185 }
3186
3187 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3188 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3189 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3190 zend_ssa_phi *phi = ssa->blocks[1].phis;
3191
3192 while (phi) {
3193 if (intervals[phi->ssa_var]) {
3194 if (intervals[phi->sources[1]]
3195 && (ssa->var_info[phi->sources[1]].type & MAY_BE_ANY) ==
3196 (ssa->var_info[phi->ssa_var].type & MAY_BE_ANY)) {
3197 intervals[phi->sources[1]]->hint = intervals[phi->ssa_var];
3198 }
3199 }
3200 phi = phi->next;
3201 }
3202 }
3203
3204 for (i = 0; i < ssa->vars_count; i++) {
3205 if (intervals[i] && !intervals[i]->hint) {
3206
3207 if (ssa->vars[i].definition >= 0) {
3208 uint32_t line = ssa->vars[i].definition;
3209 const zend_op *opline = ssa_opcodes[line];
3210
3211 switch (opline->opcode) {
3212 case ZEND_QM_ASSIGN:
3213 case ZEND_POST_INC:
3214 case ZEND_POST_DEC:
3215 if (ssa->ops[line].op1_use >= 0 &&
3216 intervals[ssa->ops[line].op1_use] &&
3217 (i == ssa->ops[line].op1_def ||
3218 (i == ssa->ops[line].result_def &&
3219 (ssa->ops[line].op1_def < 0 ||
3220 !intervals[ssa->ops[line].op1_def])))) {
3221 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3222 }
3223 break;
3224 case ZEND_SEND_VAR:
3225 if (opline->op2_type == IS_CONST) {
3226 /* Named parameters not supported in JIT */
3227 break;
3228 }
3229 case ZEND_PRE_INC:
3230 case ZEND_PRE_DEC:
3231 if (i == ssa->ops[line].op1_def &&
3232 ssa->ops[line].op1_use >= 0 &&
3233 intervals[ssa->ops[line].op1_use]) {
3234 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3235 }
3236 break;
3237 case ZEND_ASSIGN:
3238 if (ssa->ops[line].op2_use >= 0 &&
3239 intervals[ssa->ops[line].op2_use] &&
3240 (i == ssa->ops[line].op2_def ||
3241 (i == ssa->ops[line].op1_def &&
3242 (ssa->ops[line].op2_def < 0 ||
3243 !intervals[ssa->ops[line].op2_def])) ||
3244 (i == ssa->ops[line].result_def &&
3245 (ssa->ops[line].op2_def < 0 ||
3246 !intervals[ssa->ops[line].op2_def]) &&
3247 (ssa->ops[line].op1_def < 0 ||
3248 !intervals[ssa->ops[line].op1_def])))) {
3249 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3250 }
3251 break;
3252 case ZEND_SUB:
3253 case ZEND_ADD:
3254 case ZEND_MUL:
3255 case ZEND_BW_OR:
3256 case ZEND_BW_AND:
3257 case ZEND_BW_XOR:
3258 if (i == ssa->ops[line].result_def) {
3259 if (ssa->ops[line].op1_use >= 0 &&
3260 intervals[ssa->ops[line].op1_use] &&
3261 ssa->ops[line].op1_use_chain < 0 &&
3262 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
3263 (ssa->var_info[i].type & MAY_BE_ANY) ==
3264 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
3265
3266 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op1_use].definition_phi;
3267 if (phi &&
3268 intervals[phi->sources[1]] &&
3269 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op1_use]) {
3270 break;
3271 }
3272 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3273 } else if (opline->opcode != ZEND_SUB &&
3274 ssa->ops[line].op2_use >= 0 &&
3275 intervals[ssa->ops[line].op2_use] &&
3276 ssa->ops[line].op2_use_chain < 0 &&
3277 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
3278 (ssa->var_info[i].type & MAY_BE_ANY) ==
3279 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
3280
3281 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op2_use].definition_phi;
3282 if (phi &&
3283 intervals[phi->sources[1]] &&
3284 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op2_use]) {
3285 break;
3286 }
3287 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3288 }
3289 }
3290 break;
3291 }
3292 }
3293 }
3294 }
3295
3296 list = zend_jit_sort_intervals(intervals, ssa->vars_count);
3297
3298 if (list) {
3299 ival = list;
3300 while (ival) {
3301 if (ival->hint) {
3302 ival->hint->used_as_hint = ival;
3303 }
3304 ival = ival->list_next;
3305 }
3306 }
3307
3308 if (list) {
3309 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3310 fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM);
3311 ival = list;
3312 while (ival) {
3313 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3314 ival = ival->list_next;
3315 }
3316 }
3317 }
3318
3319 /* Linear Scan Register Allocation (op_array is not actually used, only fn_flags matters) */
3320 list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list);
3321
3322 if (list) {
3323 zend_lifetime_interval *ival, *next;
3324
3325 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
3326 ival = list;
3327 count = 0;
3328 while (ival != NULL) {
3329 ZEND_ASSERT(ival->reg != ZREG_NONE);
3330 count++;
3331 next = ival->list_next;
3332 ival->list_next = intervals[ival->ssa_var];
3333 intervals[ival->ssa_var] = ival;
3334 ival = next;
3335 }
3336
3337 if (!count) {
3338 zend_arena_release(&CG(arena), checkpoint);
3339 return NULL;
3340 }
3341
3342 /* Add LOAD flag to registers that can't reuse register from parent trace */
3343 if (parent_vars_count) {
3344 i = trace_buffer->op_array->last_var;
3345 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3346 i += trace_buffer->op_array->T;
3347 }
3348 if ((uint32_t)i > parent_vars_count) {
3349 i = parent_vars_count;
3350 }
3351 while (i > 0) {
3352 i--;
3353 if (intervals[i] && intervals[i]->reg != STACK_REG(parent_stack, i)) {
3354 intervals[i]->flags |= ZREG_LOAD;
3355 }
3356 }
3357 }
3358
3359 /* SSA resolution */
3360 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3361 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3362 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3363 zend_ssa_phi *phi = ssa->blocks[1].phis;
3364
3365 while (phi) {
3366 int def = phi->ssa_var;
3367 int use = phi->sources[1];
3368
3369 if (intervals[def]) {
3370 if (!intervals[use]) {
3371 intervals[def]->flags |= ZREG_LOAD;
3372 if ((intervals[def]->flags & ZREG_LAST_USE)
3373 && ssa->vars[def].use_chain >= 0
3374 && ssa->vars[def].use_chain == intervals[def]->range.end) {
3375 /* remove interval used once */
3376 intervals[def] = NULL;
3377 count--;
3378 }
3379 } else if (intervals[def]->reg != intervals[use]->reg) {
3380 intervals[def]->flags |= ZREG_LOAD;
3381 if (ssa->vars[use].use_chain >= 0) {
3382 intervals[use]->flags |= ZREG_STORE;
3383 } else {
3384 intervals[use] = NULL;
3385 count--;
3386 }
3387 } else {
3388 use = phi->sources[0];
3389 ZEND_ASSERT(!intervals[use]);
3390 intervals[use] = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
3391 intervals[use]->ssa_var = phi->sources[0];
3392 intervals[use]->reg = intervals[def]->reg;
3393 intervals[use]->flags = ZREG_LOAD;
3394 intervals[use]->range.start = 0;
3395 intervals[use]->range.end = 0;
3396 intervals[use]->range.next = NULL;
3397 intervals[use]->hint = NULL;
3398 intervals[use]->used_as_hint = NULL;
3399 intervals[use]->list_next = NULL;
3400 }
3401 } else if (intervals[use]
3402 && (!ssa->vars[def].no_val
3403 || ssa->var_info[def].type != ssa->var_info[use].type)) {
3404 if (ssa->vars[use].use_chain >= 0) {
3405 intervals[use]->flags |= ZREG_STORE;
3406 } else {
3407 intervals[use] = NULL;
3408 count--;
3409 }
3410 }
3411 phi = phi->next;
3412 }
3413 } else {
3414 for (i = 0; i < ssa->vars_count; i++) {
3415 if (intervals[i]
3416 && intervals[i]->range.end == last_idx
3417 && !(intervals[i]->flags & (ZREG_LOAD|ZREG_STORE))) {
3418 intervals[i]->flags |= ZREG_STORE;
3419 }
3420 }
3421 }
3422
3423 if (!count) {
3424 zend_arena_release(&CG(arena), checkpoint);
3425 return NULL;
3426 }
3427
3428 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3429 fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM);
3430 for (i = 0; i < ssa->vars_count; i++) {
3431 ival = intervals[i];
3432 while (ival) {
3433 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3434 ival = ival->list_next;
3435 }
3436 }
3437 }
3438
3439 return intervals;
3440 }
3441
3442 zend_arena_release(&CG(arena), checkpoint); //???
3443 return NULL;
3444 }
3445
zend_jit_trace_clenup_stack(zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,zend_lifetime_interval ** ra)3446 static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, zend_lifetime_interval **ra)
3447 {
3448 uint32_t line = ssa_op - ssa->ops;
3449
3450 if (ssa_op->op1_use >= 0
3451 && ra[ssa_op->op1_use]
3452 && ra[ssa_op->op1_use]->range.end == line) {
3453 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
3454 }
3455 if (ssa_op->op2_use >= 0
3456 && ra[ssa_op->op2_use]
3457 && ra[ssa_op->op2_use]->range.end == line) {
3458 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op2.var), ZREG_NONE);
3459 }
3460 if (ssa_op->result_use >= 0
3461 && ra[ssa_op->result_use]
3462 && ra[ssa_op->result_use]->range.end == line) {
3463 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE);
3464 }
3465 }
3466
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3467 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3468 {
3469 zend_op *next_opline = (zend_op*)(opline + 1);
3470
3471 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3472 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3473 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3474 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3475 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3476 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3477 }
3478 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3479 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3480 }
3481 }
3482
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3483 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3484 {
3485 int var = ssa_op->result_def;
3486 int i;
3487 int use = ssa->vars[var].use_chain;
3488 const zend_op *opline;
3489
3490 if (use < 0
3491 || ssa->vars[var].phi_use_chain
3492 || ssa->ops[use].op1_use != var
3493 || ssa->ops[use].op1_use_chain != -1) {
3494 return 0;
3495 }
3496
3497 opline = ssa_opcodes[use];
3498 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3499 return (opline->op2_type == IS_CONST &&
3500 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3501 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3502 if (!JIT_G(current_frame)
3503 || !JIT_G(current_frame)->call
3504 || !JIT_G(current_frame)->call->func
3505 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3506 return 0;
3507 }
3508 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3509 && opline->opcode != ZEND_FETCH_OBJ_IS
3510 && opline->opcode != ZEND_FETCH_OBJ_W
3511 && opline->opcode != ZEND_ASSIGN_OBJ
3512 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3513 && opline->opcode != ZEND_PRE_INC_OBJ
3514 && opline->opcode != ZEND_PRE_DEC_OBJ
3515 && opline->opcode != ZEND_POST_INC_OBJ
3516 && opline->opcode != ZEND_POST_DEC_OBJ) {
3517 return 0;
3518 }
3519
3520 if (opline->op2_type != IS_CONST
3521 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3522 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3523 return 0;
3524 }
3525
3526 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3527 if (opline->op1_type == IS_CV
3528 && (opline+1)->op1_type == IS_CV
3529 && (opline+1)->op1.var == opline->op1.var) {
3530 /* skip $a->prop += $a; */
3531 return 0;
3532 }
3533 if (!zend_jit_supported_binary_op(
3534 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3535 return 0;
3536 }
3537 }
3538
3539 for (i = ssa->vars[var].definition; i < use; i++) {
3540 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3541 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3542 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3543 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3544 return 0;
3545 }
3546 }
3547
3548 return 1;
3549 }
3550
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3551 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3552 {
3553 uint32_t i;
3554
3555 for (i = 0; i < stack_size; i++) {
3556 if (STACK_REG(stack, i) != ZREG_NONE
3557 && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
3558 return 1;
3559 }
3560 }
3561 return 0;
3562 }
3563
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3564 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3565 {
3566 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3567 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3568 uint32_t stack_size;
3569 zend_jit_trace_stack *stack;
3570
3571 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3572 return 1;
3573 }
3574
3575 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3576 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3577 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3578 }
3579
zend_jit_trace_deoptimization(dasm_State ** Dst,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_lifetime_interval ** ra,bool polymorphic_side_trace)3580 static int zend_jit_trace_deoptimization(dasm_State **Dst,
3581 uint32_t flags,
3582 const zend_op *opline,
3583 zend_jit_trace_stack *parent_stack,
3584 int parent_vars_count,
3585 zend_ssa *ssa,
3586 zend_jit_trace_stack *stack,
3587 zend_lifetime_interval **ra,
3588 bool polymorphic_side_trace)
3589 {
3590 int i;
3591 bool has_constants = 0;
3592 bool has_unsaved_vars = 0;
3593
3594 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3595 for (i = 0; i < parent_vars_count; i++) {
3596 int8_t reg = STACK_REG(parent_stack, i);
3597
3598 if (reg != ZREG_NONE) {
3599 if (reg < ZREG_NUM) {
3600 if (ssa && ssa->vars[i].no_val) {
3601 /* pass */
3602 } else if (ra && ra[i] && ra[i]->reg == reg) {
3603 /* register already loaded by parent trace */
3604 if (stack) {
3605 SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i));
3606 }
3607 has_unsaved_vars = 1;
3608 } else {
3609 uint8_t type = STACK_TYPE(parent_stack, i);
3610
3611 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3612 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3613 STACK_MEM_TYPE(parent_stack, i) != type)) {
3614 return 0;
3615 }
3616 if (stack) {
3617 SET_STACK_TYPE(stack, i, type, 1);
3618 }
3619 }
3620 } else {
3621 /* delay custom deoptimization instructions to prevent register clobbering */
3622 has_constants = 1;
3623 }
3624 }
3625 }
3626
3627 if (has_unsaved_vars
3628 && (has_constants
3629 || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) {
3630 for (i = 0; i < parent_vars_count; i++) {
3631 int8_t reg = STACK_REG(parent_stack, i);
3632
3633 if (reg != ZREG_NONE) {
3634 if (reg < ZREG_NUM) {
3635 if (ssa && ssa->vars[i].no_val) {
3636 /* pass */
3637 } else if (ra && ra[i] && ra[i]->reg == reg) {
3638 uint8_t type = STACK_TYPE(parent_stack, i);
3639
3640 if (stack) {
3641 SET_STACK_TYPE(stack, i, type, 1);
3642 }
3643 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3644 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3645 STACK_MEM_TYPE(parent_stack, i) != type)) {
3646 return 0;
3647 }
3648 }
3649 }
3650 }
3651 }
3652 }
3653
3654 if (has_constants) {
3655 for (i = 0; i < parent_vars_count; i++) {
3656 int8_t reg = STACK_REG(parent_stack, i);
3657
3658 if (reg != ZREG_NONE) {
3659 if (reg < ZREG_NUM) {
3660 /* pass */
3661 } else if (reg == ZREG_THIS) {
3662 if (polymorphic_side_trace) {
3663 ssa->var_info[i].delayed_fetch_this = 1;
3664 if (stack) {
3665 SET_STACK_REG(stack, i, ZREG_THIS);
3666 }
3667 } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
3668 return 0;
3669 }
3670 } else {
3671 if (reg == ZREG_ZVAL_COPY_GPR0
3672 &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) {
3673 return 0;
3674 }
3675 if (!zend_jit_store_const(Dst, i, reg)) {
3676 return 0;
3677 }
3678 }
3679 }
3680 }
3681 }
3682
3683 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3684 if (!zend_jit_save_call_chain(Dst, -1)) {
3685 return 0;
3686 }
3687 }
3688
3689 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3690 const zend_op *op = opline - 1;
3691
3692 if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) {
3693 return 0;
3694 }
3695 }
3696
3697 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3698 const zend_op *op = opline - 1;
3699
3700 if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) {
3701 return 0;
3702 }
3703 }
3704
3705 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3706 if (!zend_jit_check_exception(Dst)) {
3707 return 0;
3708 }
3709 }
3710
3711 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3712 if (!zend_jit_free_trampoline(Dst)) {
3713 return 0;
3714 }
3715 }
3716
3717 return 1;
3718 }
3719
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3720 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3721 {
3722 info->has_range = 1;
3723 info->range.min = min;
3724 info->range.max = max;
3725 info->range.underflow = 0;
3726 info->range.overflow = 0;
3727 }
3728
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3729 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3730 {
3731 zend_long op1_min, op1_max, op2_min, op2_max;
3732
3733 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3734 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3735 return;
3736 }
3737
3738 op1_min = OP1_MIN_RANGE();
3739 op1_max = OP1_MAX_RANGE();
3740 op2_min = OP2_MIN_RANGE();
3741 op2_max = OP2_MAX_RANGE();
3742
3743 switch (opline->opcode) {
3744 case ZEND_IS_EQUAL:
3745 case ZEND_CASE:
3746 case ZEND_IS_IDENTICAL:
3747 case ZEND_CASE_STRICT:
3748 case ZEND_IS_NOT_IDENTICAL:
3749 if (!exit_if_true) {
3750 /* op1 == op2 */
3751 if (ssa_op->op1_use >= 0) {
3752 zend_jit_trace_set_var_range(
3753 &ssa->var_info[ssa_op->op1_use],
3754 MAX(op1_min, op2_min),
3755 MIN(op1_max, op2_max));
3756 }
3757 if (ssa_op->op2_use >= 0) {
3758 zend_jit_trace_set_var_range(
3759 &ssa->var_info[ssa_op->op2_use],
3760 MAX(op2_min, op1_min),
3761 MIN(op2_max, op1_max));
3762 }
3763 }
3764 break;
3765 case ZEND_IS_NOT_EQUAL:
3766 if (exit_if_true) {
3767 /* op1 == op2 */
3768 if (ssa_op->op1_use >= 0) {
3769 zend_jit_trace_set_var_range(
3770 &ssa->var_info[ssa_op->op1_use],
3771 MAX(op1_min, op2_min),
3772 MIN(op1_max, op2_max));
3773 }
3774 if (ssa_op->op2_use >= 0) {
3775 zend_jit_trace_set_var_range(
3776 &ssa->var_info[ssa_op->op2_use],
3777 MAX(op2_min, op1_min),
3778 MIN(op2_max, op1_max));
3779 }
3780 }
3781 break;
3782 case ZEND_IS_SMALLER_OR_EQUAL:
3783 if (!exit_if_true) {
3784 /* op1 <= op2 */
3785 if (ssa_op->op1_use >= 0) {
3786 zend_jit_trace_set_var_range(
3787 &ssa->var_info[ssa_op->op1_use],
3788 op1_min,
3789 MIN(op1_max, op2_max));
3790 }
3791 if (ssa_op->op2_use >= 0) {
3792 zend_jit_trace_set_var_range(
3793 &ssa->var_info[ssa_op->op2_use],
3794 MAX(op2_min, op1_min),
3795 op2_max);
3796 }
3797 } else {
3798 /* op1 > op2 */
3799 if (ssa_op->op1_use >= 0) {
3800 zend_jit_trace_set_var_range(
3801 &ssa->var_info[ssa_op->op1_use],
3802 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3803 op1_max);
3804 }
3805 if (ssa_op->op2_use >= 0) {
3806 zend_jit_trace_set_var_range(
3807 &ssa->var_info[ssa_op->op2_use],
3808 op2_min,
3809 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3810 }
3811 }
3812 break;
3813 case ZEND_IS_SMALLER:
3814 if (!exit_if_true) {
3815 /* op1 < op2 */
3816 if (ssa_op->op1_use >= 0) {
3817 zend_jit_trace_set_var_range(
3818 &ssa->var_info[ssa_op->op1_use],
3819 op1_min,
3820 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3821 }
3822 if (ssa_op->op2_use >= 0) {
3823 zend_jit_trace_set_var_range(
3824 &ssa->var_info[ssa_op->op2_use],
3825 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3826 op2_max);
3827 }
3828 } else {
3829 /* op1 >= op2 */
3830 if (ssa_op->op1_use >= 0) {
3831 zend_jit_trace_set_var_range(
3832 &ssa->var_info[ssa_op->op1_use],
3833 MAX(op1_min, op2_min),
3834 op1_max);
3835 }
3836 if (ssa_op->op2_use >= 0) {
3837 zend_jit_trace_set_var_range(
3838 &ssa->var_info[ssa_op->op2_use],
3839 op2_min,
3840 MIN(op2_max, op1_max));
3841 }
3842 }
3843 break;
3844 }
3845 }
3846
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3847 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3848 {
3849 zend_uchar prev_opcode;
3850
3851 if (opline->op1_type == IS_CONST
3852 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3853 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3854 if (ssa_op->op2_use >= 0) {
3855 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3856 ssa_op--;
3857 opline = ssa_opcodes[ssa_op - ssa->ops];
3858 prev_opcode = opline->opcode;
3859 if (prev_opcode == ZEND_PRE_INC
3860 || prev_opcode == ZEND_PRE_DEC
3861 || prev_opcode == ZEND_POST_INC
3862 || prev_opcode == ZEND_POST_DEC) {
3863 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3864 }
3865 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3866 ssa_op--;
3867 opline = ssa_opcodes[ssa_op - ssa->ops];
3868 prev_opcode = opline->opcode;
3869 if (prev_opcode == ZEND_ADD
3870 || prev_opcode == ZEND_SUB) {
3871 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3872 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3873 }
3874 }
3875 }
3876 } else if (opline->op2_type == IS_CONST
3877 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3878 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3879 if (ssa_op->op1_use >= 0) {
3880 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3881 ssa_op--;
3882 opline = ssa_opcodes[ssa_op - ssa->ops];
3883 prev_opcode = opline->opcode;
3884 if (prev_opcode == ZEND_PRE_INC
3885 || prev_opcode == ZEND_PRE_DEC
3886 || prev_opcode == ZEND_POST_INC
3887 || prev_opcode == ZEND_POST_DEC) {
3888 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3889 }
3890 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3891 ssa_op--;
3892 opline = ssa_opcodes[ssa_op - ssa->ops];
3893 prev_opcode = opline->opcode;
3894 if (prev_opcode == ZEND_ADD
3895 || prev_opcode == ZEND_SUB) {
3896 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3897 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3898 }
3899 }
3900 }
3901 } else {
3902 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3903 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3904
3905 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3906 && prev_ssa_op != ssa->ops
3907 && prev_ssa_op->op1_use >= 0
3908 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3909 prev_ssa_op--;
3910 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3911 }
3912
3913 if (ssa_op->op1_use == prev_ssa_op->op1_use
3914 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3915 if (prev_opcode == ZEND_IS_EQUAL
3916 || prev_opcode == ZEND_IS_NOT_EQUAL
3917 || prev_opcode == ZEND_IS_SMALLER
3918 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3919 || prev_opcode == ZEND_CASE
3920 || prev_opcode == ZEND_IS_IDENTICAL
3921 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3922 || prev_opcode == ZEND_CASE_STRICT) {
3923 if (ssa_op->op1_use < 0) {
3924 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3925 return 0;
3926 }
3927 }
3928 if (ssa_op->op2_use < 0) {
3929 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3930 return 0;
3931 }
3932 }
3933 return 1;
3934 }
3935 }
3936 }
3937 return 0;
3938 }
3939
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3940 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3941 zend_jit_trace_rec *p,
3942 zend_jit_trace_stack_frame *frame)
3943 {
3944 if (opline->result_type == IS_TMP_VAR
3945 && (p+1)->op == ZEND_JIT_TRACE_VM
3946 && (p+1)->opline == opline + 1
3947 && ((opline+1)->opcode == ZEND_SEND_VAL
3948 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3949 && frame
3950 && frame->call
3951 && frame->call->func
3952 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3953 && (opline+1)->op1_type == IS_TMP_VAR
3954 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3955 && (opline+1)->op1.var == opline->result.var) {
3956
3957 if (frame->call && frame->call->func) {
3958 uint8_t res_type = (p+1)->op1_type;
3959
3960 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3961 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3962 }
3963 }
3964 return 1;
3965 }
3966 return 0;
3967 }
3968
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)3969 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
3970 {
3971 const void *handler = NULL;
3972 dasm_State* dasm_state = NULL;
3973 zend_script *script = NULL;
3974 zend_lifetime_interval **ra = NULL;
3975 zend_string *name = NULL;
3976 void *checkpoint;
3977 const zend_op_array *op_array;
3978 zend_ssa *ssa, *op_array_ssa;
3979 const zend_op **ssa_opcodes;
3980 zend_jit_trace_rec *p;
3981 zend_jit_op_array_trace_extension *jit_extension;
3982 int num_op_arrays = 0;
3983 zend_jit_trace_info *t;
3984 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
3985 zend_uchar smart_branch_opcode;
3986 const void *exit_addr;
3987 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info;
3988 bool send_result = 0;
3989 bool skip_comparison;
3990 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
3991 zend_class_entry *ce;
3992 bool ce_is_instanceof;
3993 bool on_this = 0;
3994 bool delayed_fetch_this = 0;
3995 bool avoid_refcounting = 0;
3996 bool polymorphic_side_trace =
3997 parent_trace &&
3998 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
3999 uint32_t i;
4000 zend_jit_trace_stack_frame *frame, *top, *call;
4001 zend_jit_trace_stack *stack;
4002 zend_uchar res_type = IS_UNKNOWN;
4003 const zend_op *opline, *orig_opline;
4004 const zend_ssa_op *ssa_op, *orig_ssa_op;
4005 int checked_stack;
4006 int peek_checked_stack;
4007 uint32_t frame_flags = 0;
4008
4009 JIT_G(current_trace) = trace_buffer;
4010
4011 checkpoint = zend_arena_checkpoint(CG(arena));
4012
4013 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4014
4015 if (!ssa) {
4016 goto jit_cleanup;
4017 }
4018
4019 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4020
4021 /* Register allocation */
4022 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4023 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4024 ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4025 }
4026
4027 p = trace_buffer;
4028 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4029 op_array = p->op_array;
4030 frame = JIT_G(current_frame);
4031 top = zend_jit_trace_call_frame(frame, op_array);
4032 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4033 frame->used_stack = checked_stack = peek_checked_stack = 0;
4034 stack = frame->stack;
4035 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4036 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4037 }
4038
4039 opline = p[1].opline;
4040 name = zend_jit_trace_name(op_array, opline->lineno);
4041 p += ZEND_JIT_TRACE_START_REC_SIZE;
4042
4043 dasm_init(&dasm_state, DASM_MAXSECTION);
4044 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4045 dasm_setup(&dasm_state, dasm_actions);
4046
4047 jit_extension =
4048 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4049 op_array_ssa = &jit_extension->func_info.ssa;
4050
4051 dasm_growpc(&dasm_state, 2); /* =>0: loop header */
4052 /* =>1: end of code */
4053
4054 zend_jit_align_func(&dasm_state);
4055 if (!parent_trace) {
4056 zend_jit_prologue(&dasm_state);
4057 }
4058 zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM,
4059 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4060
4061 if (!parent_trace) {
4062 zend_jit_set_last_valid_opline(opline);
4063 zend_jit_track_last_valid_opline();
4064 } else {
4065 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4066 zend_jit_trace_opline_guard(&dasm_state, opline);
4067 } else {
4068 zend_jit_reset_last_valid_opline();
4069 }
4070 }
4071
4072 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4073 int last_var;
4074 int parent_vars_count = 0;
4075 zend_jit_trace_stack *parent_stack = NULL;
4076 int used_stack = ((zend_tssa*)ssa)->used_stack;
4077
4078 if (used_stack > 0) {
4079 peek_checked_stack = used_stack;
4080 if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) {
4081 goto jit_failure;
4082 }
4083 }
4084
4085 if (parent_trace) {
4086 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4087 op_array->last_var + op_array->T);
4088 if (parent_vars_count) {
4089 parent_stack =
4090 zend_jit_traces[parent_trace].stack_map +
4091 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4092 }
4093 }
4094
4095 last_var = op_array->last_var;
4096 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4097 last_var += op_array->T;
4098 }
4099
4100 for (i = 0; i < last_var; i++) {
4101 uint32_t info = ssa->var_info[i].type;
4102
4103 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4104 uint8_t type, mem_type;
4105
4106 type = concrete_type(info);
4107 if (i < parent_vars_count
4108 && STACK_TYPE(parent_stack, i) == type) {
4109 mem_type = STACK_MEM_TYPE(parent_stack, i);
4110 if (mem_type != IS_UNKNOWN) {
4111 SET_STACK_TYPE(stack, i, mem_type, 1);
4112 }
4113 SET_STACK_TYPE(stack, i, type, 0);
4114 } else {
4115 SET_STACK_TYPE(stack, i, type, 1);
4116 }
4117 } else if (ssa->vars[i].alias != NO_ALIAS) {
4118 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4119 } else if (i < parent_vars_count
4120 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4121 /* This must be already handled by trace type inference */
4122 ZEND_UNREACHABLE();
4123 // SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
4124 } else if ((info & MAY_BE_GUARD) != 0
4125 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4126 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4127 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4128 && (opline-1)->result_type == IS_VAR
4129 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4130 && (ssa->vars[i].use_chain != -1
4131 || (ssa->vars[i].phi_use_chain
4132 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4133 /* Check loop-invariant variable type */
4134 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4135 goto jit_failure;
4136 }
4137 info &= ~MAY_BE_GUARD;
4138 ssa->var_info[i].type = info;
4139 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4140 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4141 && op_array->function_name
4142 && i >= op_array->num_args) {
4143 /* This must be already handled by trace type inference */
4144 ZEND_UNREACHABLE();
4145 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4146 }
4147
4148 if ((info & MAY_BE_PACKED_GUARD) != 0
4149 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4150 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4151 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4152 && (ssa->vars[i].use_chain != -1
4153 || (ssa->vars[i].phi_use_chain
4154 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4155 if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) {
4156 goto jit_failure;
4157 }
4158 info &= ~MAY_BE_PACKED_GUARD;
4159 ssa->var_info[i].type = info;
4160 }
4161 }
4162
4163 if (parent_trace) {
4164 /* Deoptimization */
4165 if (!zend_jit_trace_deoptimization(&dasm_state,
4166 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4167 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4168 parent_stack, parent_vars_count, ssa, stack, ra,
4169 polymorphic_side_trace)) {
4170 goto jit_failure;
4171 }
4172 }
4173
4174 if (ra
4175 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4176 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4177 for (i = 0; i < last_var; i++) {
4178 if (ra[i]
4179 && (ra[i]->flags & ZREG_LOAD) != 0
4180 && ra[i]->reg != stack[i].reg) {
4181
4182 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4183 uint8_t op_type;
4184
4185 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4186 op_type = concrete_type(ssa->var_info[i].type);
4187 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
4188 goto jit_failure;
4189 }
4190 SET_STACK_TYPE(stack, i, op_type, 1);
4191 }
4192
4193 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4194 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4195 goto jit_failure;
4196 }
4197 }
4198 }
4199 }
4200 }
4201
4202 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4203 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4204 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4205
4206 zend_jit_label(&dasm_state, 0); /* start of of trace loop */
4207
4208 if (ra) {
4209 zend_ssa_phi *phi = ssa->blocks[1].phis;
4210
4211 while (phi) {
4212 zend_lifetime_interval *ival = ra[phi->ssa_var];
4213
4214 if (ival) {
4215 if (ival->flags & ZREG_LOAD) {
4216 uint32_t info = ssa->var_info[phi->ssa_var].type;
4217 ZEND_ASSERT(ival->reg != ZREG_NONE);
4218
4219 if (info & MAY_BE_GUARD) {
4220 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4221 goto jit_failure;
4222 }
4223 info &= ~MAY_BE_GUARD;
4224 ssa->var_info[phi->ssa_var].type = info;
4225 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4226 }
4227 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_LOAD);
4228 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
4229 goto jit_failure;
4230 }
4231 } else if (ival->flags & ZREG_STORE) {
4232 ZEND_ASSERT(ival->reg != ZREG_NONE);
4233
4234 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_STORE);
4235 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg,
4236 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4237 goto jit_failure;
4238 }
4239 } else {
4240 /* Register has to be written back on side exit */
4241 SET_STACK_REG(stack, phi->var, ival->reg);
4242 }
4243 }
4244 phi = phi->next;
4245 }
4246 }
4247
4248 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4249 // if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4250 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4251 //
4252 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4253 // if (!timeout_exit_addr) {
4254 // goto jit_failure;
4255 // }
4256 // }
4257 // }
4258
4259 if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
4260 int last_var = op_array->last_var;
4261
4262 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4263 last_var += op_array->T;
4264 }
4265 for (i = 0; i < last_var; i++) {
4266 if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) {
4267 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4268 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4269 goto jit_failure;
4270 }
4271 }
4272 }
4273 }
4274 }
4275
4276 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4277 for (;;p++) {
4278 if (p->op == ZEND_JIT_TRACE_VM) {
4279 uint8_t op1_type = p->op1_type;
4280 uint8_t op2_type = p->op2_type;
4281 uint8_t op3_type = p->op3_type;
4282 uint8_t orig_op1_type = op1_type;
4283 uint8_t orig_op2_type = op2_type;
4284 uint8_t val_type = IS_UNKNOWN;
4285 bool op1_indirect;
4286 zend_class_entry *op1_ce = NULL;
4287 zend_class_entry *op2_ce = NULL;
4288 bool gen_handler;
4289
4290 opline = p->opline;
4291 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4292 op1_type = IS_UNKNOWN;
4293 }
4294 if (op1_type != IS_UNKNOWN) {
4295 op1_type &= ~IS_TRACE_PACKED;
4296 }
4297 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4298 op2_type = IS_UNKNOWN;
4299 }
4300 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4301 op3_type = IS_UNKNOWN;
4302 }
4303
4304 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4305 op1_ce = (zend_class_entry*)(p+1)->ce;
4306 p++;
4307 }
4308 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4309 op2_ce = (zend_class_entry*)(p+1)->ce;
4310 p++;
4311 }
4312 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4313 val_type = (p+1)->op1_type;
4314 p++;
4315 }
4316
4317 frame_flags = 0;
4318
4319 switch (opline->opcode) {
4320 case ZEND_INIT_FCALL:
4321 case ZEND_INIT_FCALL_BY_NAME:
4322 case ZEND_INIT_NS_FCALL_BY_NAME:
4323 case ZEND_INIT_METHOD_CALL:
4324 case ZEND_INIT_DYNAMIC_CALL:
4325 case ZEND_INIT_STATIC_METHOD_CALL:
4326 case ZEND_INIT_USER_CALL:
4327 case ZEND_NEW:
4328 frame->call_level++;
4329 }
4330
4331 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4332 gen_handler = 0;
4333 switch (opline->opcode) {
4334 case ZEND_PRE_INC:
4335 case ZEND_PRE_DEC:
4336 case ZEND_POST_INC:
4337 case ZEND_POST_DEC:
4338 if (opline->op1_type != IS_CV) {
4339 break;
4340 }
4341 op1_info = OP1_INFO();
4342 CHECK_OP1_TRACE_TYPE();
4343 if (!(op1_info & MAY_BE_LONG)) {
4344 break;
4345 }
4346 if (opline->result_type != IS_UNUSED) {
4347 res_use_info = zend_jit_trace_type_to_info(
4348 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4349 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4350 res_info = RES_INFO();
4351 res_addr = RES_REG_ADDR();
4352 } else {
4353 res_use_info = -1;
4354 res_info = -1;
4355 res_addr = 0;
4356 }
4357 op1_def_info = OP1_DEF_INFO();
4358 if (op1_def_info & MAY_BE_GUARD
4359 && !has_concrete_type(op1_def_info)) {
4360 op1_def_info &= ~MAY_BE_GUARD;
4361 }
4362 if (!zend_jit_inc_dec(&dasm_state, opline,
4363 op1_info, OP1_REG_ADDR(),
4364 op1_def_info, OP1_DEF_REG_ADDR(),
4365 res_use_info, res_info,
4366 res_addr,
4367 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4368 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4369 goto jit_failure;
4370 }
4371 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4372 && !(op1_info & MAY_BE_STRING)) {
4373 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4374 if (opline->result_type != IS_UNUSED) {
4375 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4376 }
4377 }
4378 if (opline->result_type != IS_UNUSED
4379 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4380 && !(op1_info & MAY_BE_STRING)) {
4381 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4382 }
4383 goto done;
4384 case ZEND_BW_OR:
4385 case ZEND_BW_AND:
4386 case ZEND_BW_XOR:
4387 case ZEND_SL:
4388 case ZEND_SR:
4389 case ZEND_MOD:
4390 op1_info = OP1_INFO();
4391 CHECK_OP1_TRACE_TYPE();
4392 op2_info = OP2_INFO();
4393 CHECK_OP2_TRACE_TYPE();
4394 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4395 break;
4396 }
4397 if (!(op1_info & MAY_BE_LONG)
4398 || !(op2_info & MAY_BE_LONG)) {
4399 break;
4400 }
4401 res_addr = RES_REG_ADDR();
4402 if (Z_MODE(res_addr) != IS_REG
4403 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4404 send_result = 1;
4405 res_use_info = -1;
4406 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4407 if (!zend_jit_reuse_ip(&dasm_state)) {
4408 goto jit_failure;
4409 }
4410 } else {
4411 res_use_info = zend_jit_trace_type_to_info(
4412 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4413 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4414 }
4415 res_info = RES_INFO();
4416 if (!zend_jit_long_math(&dasm_state, opline,
4417 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4418 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4419 res_use_info, res_info, res_addr,
4420 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4421 goto jit_failure;
4422 }
4423 goto done;
4424 case ZEND_ADD:
4425 case ZEND_SUB:
4426 case ZEND_MUL:
4427 // case ZEND_DIV: // TODO: check for division by zero ???
4428 op1_info = OP1_INFO();
4429 op1_addr = OP1_REG_ADDR();
4430 op2_info = OP2_INFO();
4431 op2_addr = OP2_REG_ADDR();
4432 if (orig_op1_type != IS_UNKNOWN
4433 && (orig_op1_type & IS_TRACE_REFERENCE)
4434 && opline->op1_type == IS_CV
4435 && (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_FCARG1)
4436 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4437 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4438 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4439 goto jit_failure;
4440 }
4441 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4442 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4443 }
4444 } else {
4445 CHECK_OP1_TRACE_TYPE();
4446 }
4447 if (orig_op2_type != IS_UNKNOWN
4448 && (orig_op2_type & IS_TRACE_REFERENCE)
4449 && opline->op2_type == IS_CV
4450 && (Z_MODE(op1_addr) != IS_REG || Z_REG(op1_addr) != ZREG_FCARG1)
4451 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4452 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op2_type, &op2_info, &op2_addr,
4453 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4454 goto jit_failure;
4455 }
4456 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4457 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4458 }
4459 } else {
4460 CHECK_OP2_TRACE_TYPE();
4461 }
4462 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4463 break;
4464 }
4465 if (opline->opcode == ZEND_ADD &&
4466 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4467 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4468 /* pass */
4469 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4470 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4471 break;
4472 }
4473 res_addr = RES_REG_ADDR();
4474 if (Z_MODE(res_addr) != IS_REG
4475 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4476 send_result = 1;
4477 res_use_info = -1;
4478 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4479 if (!zend_jit_reuse_ip(&dasm_state)) {
4480 goto jit_failure;
4481 }
4482 } else {
4483 res_use_info = zend_jit_trace_type_to_info(
4484 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4485 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4486 }
4487 res_info = RES_INFO();
4488 if (opline->opcode == ZEND_ADD &&
4489 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4490 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4491 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4492 goto jit_failure;
4493 }
4494 } else {
4495 if (!zend_jit_math(&dasm_state, opline,
4496 op1_info, op1_addr,
4497 op2_info, op2_addr,
4498 res_use_info, res_info, res_addr,
4499 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4500 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4501 goto jit_failure;
4502 }
4503 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4504 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4505 && has_concrete_type(op1_info)
4506 && has_concrete_type(op2_info)) {
4507 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4508 }
4509 }
4510 goto done;
4511 case ZEND_CONCAT:
4512 case ZEND_FAST_CONCAT:
4513 op1_info = OP1_INFO();
4514 CHECK_OP1_TRACE_TYPE();
4515 op2_info = OP2_INFO();
4516 CHECK_OP2_TRACE_TYPE();
4517 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4518 break;
4519 }
4520 if (!(op1_info & MAY_BE_STRING) ||
4521 !(op2_info & MAY_BE_STRING)) {
4522 break;
4523 }
4524 res_addr = RES_REG_ADDR();
4525 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4526 send_result = 1;
4527 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4528 if (!zend_jit_reuse_ip(&dasm_state)) {
4529 goto jit_failure;
4530 }
4531 }
4532 if (!zend_jit_concat(&dasm_state, opline,
4533 op1_info, op2_info, res_addr,
4534 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4535 goto jit_failure;
4536 }
4537 goto done;
4538 case ZEND_ASSIGN_OP:
4539 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4540 break;
4541 }
4542 op1_info = OP1_INFO();
4543 CHECK_OP1_TRACE_TYPE();
4544 op2_info = OP2_INFO();
4545 CHECK_OP2_TRACE_TYPE();
4546 if (!zend_jit_supported_binary_op(
4547 opline->extended_value, op1_info, op2_info)) {
4548 break;
4549 }
4550 op1_def_info = OP1_DEF_INFO();
4551 if (op1_def_info & MAY_BE_GUARD
4552 && !has_concrete_type(op1_def_info)) {
4553 op1_def_info &= ~MAY_BE_GUARD;
4554 }
4555 if (!zend_jit_assign_op(&dasm_state, opline,
4556 op1_info, op1_def_info, OP1_RANGE(),
4557 op2_info, OP2_RANGE(),
4558 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4559 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4560 goto jit_failure;
4561 }
4562 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4563 && has_concrete_type(op1_info)
4564 && has_concrete_type(op2_info)) {
4565 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4566 if (opline->result_type != IS_UNUSED) {
4567 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4568 }
4569 }
4570 goto done;
4571 case ZEND_ASSIGN_DIM_OP:
4572 if (opline->result_type != IS_UNUSED) {
4573 break;
4574 }
4575 if (!zend_jit_supported_binary_op(
4576 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4577 break;
4578 }
4579 if (opline->op1_type == IS_CV
4580 && (opline+1)->op1_type == IS_CV
4581 && (opline+1)->op1.var == opline->op1.var) {
4582 /* skip $a[x] += $a; */
4583 break;
4584 }
4585 op1_info = OP1_INFO();
4586 op1_addr = OP1_REG_ADDR();
4587 if (opline->op1_type == IS_VAR) {
4588 if (orig_op1_type != IS_UNKNOWN
4589 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4590 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4591 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4592 goto jit_failure;
4593 }
4594 } else {
4595 break;
4596 }
4597 }
4598 if (orig_op1_type != IS_UNKNOWN
4599 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4600 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4601 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4602 goto jit_failure;
4603 }
4604 if (opline->op1_type == IS_CV
4605 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4606 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4607 }
4608 } else {
4609 CHECK_OP1_TRACE_TYPE();
4610 }
4611 op2_info = OP2_INFO();
4612 CHECK_OP2_TRACE_TYPE();
4613 op1_data_info = OP1_DATA_INFO();
4614 CHECK_OP1_DATA_TRACE_TYPE();
4615 op1_def_info = OP1_DEF_INFO();
4616 if (!zend_jit_assign_dim_op(&dasm_state, opline,
4617 op1_info, op1_def_info, op1_addr, op2_info,
4618 op1_data_info, OP1_DATA_RANGE(), val_type,
4619 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4620 goto jit_failure;
4621 }
4622 goto done;
4623 case ZEND_PRE_INC_OBJ:
4624 case ZEND_PRE_DEC_OBJ:
4625 case ZEND_POST_INC_OBJ:
4626 case ZEND_POST_DEC_OBJ:
4627 if (opline->op2_type != IS_CONST
4628 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4629 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4630 break;
4631 }
4632 ce = NULL;
4633 ce_is_instanceof = 0;
4634 on_this = delayed_fetch_this = 0;
4635 op1_indirect = 0;
4636 if (opline->op1_type == IS_UNUSED) {
4637 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4638 ce = op_array->scope;
4639 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4640 op1_addr = 0;
4641 on_this = 1;
4642 } else {
4643 if (ssa_op->op1_use >= 0) {
4644 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4645 }
4646 op1_info = OP1_INFO();
4647 if (!(op1_info & MAY_BE_OBJECT)) {
4648 break;
4649 }
4650 op1_addr = OP1_REG_ADDR();
4651 if (opline->op1_type == IS_VAR) {
4652 if (orig_op1_type != IS_UNKNOWN
4653 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4654 op1_indirect = 1;
4655 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4656 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4657 goto jit_failure;
4658 }
4659 }
4660 }
4661 if (orig_op1_type != IS_UNKNOWN
4662 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4663 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4664 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4665 goto jit_failure;
4666 }
4667 if (opline->op1_type == IS_CV
4668 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4669 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4670 }
4671 } else {
4672 CHECK_OP1_TRACE_TYPE();
4673 }
4674 if (!(op1_info & MAY_BE_OBJECT)) {
4675 break;
4676 }
4677 if (ssa->var_info && ssa->ops) {
4678 if (ssa_op->op1_use >= 0) {
4679 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4680 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4681 ce = op1_ssa->ce;
4682 ce_is_instanceof = op1_ssa->is_instanceof;
4683 }
4684 }
4685 }
4686 if (delayed_fetch_this) {
4687 on_this = 1;
4688 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4689 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4690 } else if (op_array_ssa->ops
4691 && op_array_ssa->vars
4692 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4693 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4694 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4695 }
4696 }
4697 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4698 op1_info, op1_addr,
4699 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4700 val_type)) {
4701 goto jit_failure;
4702 }
4703 goto done;
4704 case ZEND_ASSIGN_OBJ_OP:
4705 if (opline->result_type != IS_UNUSED) {
4706 break;
4707 }
4708 if (opline->op2_type != IS_CONST
4709 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4710 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4711 break;
4712 }
4713 if (opline->op1_type == IS_CV
4714 && (opline+1)->op1_type == IS_CV
4715 && (opline+1)->op1.var == opline->op1.var) {
4716 /* skip $a->prop += $a; */
4717 break;
4718 }
4719 if (!zend_jit_supported_binary_op(
4720 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4721 break;
4722 }
4723 ce = NULL;
4724 ce_is_instanceof = 0;
4725 on_this = delayed_fetch_this = 0;
4726 op1_indirect = 0;
4727 if (opline->op1_type == IS_UNUSED) {
4728 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4729 ce = op_array->scope;
4730 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4731 op1_addr = 0;
4732 on_this = 1;
4733 } else {
4734 if (ssa_op->op1_use >= 0) {
4735 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4736 }
4737 op1_info = OP1_INFO();
4738 if (!(op1_info & MAY_BE_OBJECT)) {
4739 break;
4740 }
4741 op1_addr = OP1_REG_ADDR();
4742 if (opline->op1_type == IS_VAR) {
4743 if (orig_op1_type != IS_UNKNOWN
4744 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4745 op1_indirect = 1;
4746 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4747 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4748 goto jit_failure;
4749 }
4750 }
4751 }
4752 if (orig_op1_type != IS_UNKNOWN
4753 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4754 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4755 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4756 goto jit_failure;
4757 }
4758 if (opline->op1_type == IS_CV
4759 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4760 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4761 }
4762 } else {
4763 CHECK_OP1_TRACE_TYPE();
4764 }
4765 if (!(op1_info & MAY_BE_OBJECT)) {
4766 break;
4767 }
4768 if (ssa->var_info && ssa->ops) {
4769 if (ssa_op->op1_use >= 0) {
4770 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4771 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4772 ce = op1_ssa->ce;
4773 ce_is_instanceof = op1_ssa->is_instanceof;
4774 }
4775 }
4776 }
4777 if (delayed_fetch_this) {
4778 on_this = 1;
4779 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4780 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4781 } else if (op_array_ssa->ops
4782 && op_array_ssa->vars
4783 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4784 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4785 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4786 }
4787 }
4788 op1_data_info = OP1_DATA_INFO();
4789 CHECK_OP1_DATA_TRACE_TYPE();
4790 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
4791 op1_info, op1_addr, op1_data_info, OP1_DATA_RANGE(),
4792 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4793 val_type)) {
4794 goto jit_failure;
4795 }
4796 goto done;
4797 case ZEND_ASSIGN_OBJ:
4798 if (opline->op2_type != IS_CONST
4799 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4800 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4801 break;
4802 }
4803 ce = NULL;
4804 ce_is_instanceof = 0;
4805 on_this = delayed_fetch_this = 0;
4806 op1_indirect = 0;
4807 if (opline->op1_type == IS_UNUSED) {
4808 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4809 ce = op_array->scope;
4810 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4811 op1_addr = 0;
4812 on_this = 1;
4813 } else {
4814 if (ssa_op->op1_use >= 0) {
4815 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4816 }
4817 op1_info = OP1_INFO();
4818 if (!(op1_info & MAY_BE_OBJECT)) {
4819 break;
4820 }
4821 op1_addr = OP1_REG_ADDR();
4822 if (opline->op1_type == IS_VAR) {
4823 if (orig_op1_type != IS_UNKNOWN
4824 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4825 op1_indirect = 1;
4826 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4827 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4828 goto jit_failure;
4829 }
4830 }
4831 }
4832 if (orig_op1_type != IS_UNKNOWN
4833 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4834 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4835 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4836 goto jit_failure;
4837 }
4838 if (opline->op1_type == IS_CV
4839 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4840 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4841 }
4842 } else {
4843 CHECK_OP1_TRACE_TYPE();
4844 }
4845 if (!(op1_info & MAY_BE_OBJECT)) {
4846 break;
4847 }
4848 if (ssa->var_info && ssa->ops) {
4849 if (ssa_op->op1_use >= 0) {
4850 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4851 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4852 ce = op1_ssa->ce;
4853 ce_is_instanceof = op1_ssa->is_instanceof;
4854 }
4855 }
4856 }
4857 if (delayed_fetch_this) {
4858 on_this = 1;
4859 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4860 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4861 } else if (op_array_ssa->ops
4862 && op_array_ssa->vars
4863 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4864 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4865 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4866 }
4867 }
4868 op1_data_info = OP1_DATA_INFO();
4869 CHECK_OP1_DATA_TRACE_TYPE();
4870 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4871 op1_info, op1_addr, op1_data_info,
4872 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4873 val_type,
4874 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4875 goto jit_failure;
4876 }
4877 if ((opline+1)->op1_type == IS_CV
4878 && (ssa_op+1)->op1_def >= 0
4879 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4880 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4881 }
4882 goto done;
4883 case ZEND_ASSIGN_DIM:
4884 op1_info = OP1_INFO();
4885 op1_addr = OP1_REG_ADDR();
4886 if (opline->op1_type == IS_CV
4887 && (opline+1)->op1_type == IS_CV
4888 && (opline+1)->op1.var == opline->op1.var) {
4889 /* skip $a[x] = $a; */
4890 break;
4891 }
4892 if (opline->op1_type == IS_VAR) {
4893 if (orig_op1_type != IS_UNKNOWN
4894 && (orig_op1_type & IS_TRACE_INDIRECT)
4895 && opline->result_type == IS_UNUSED) {
4896 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4897 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4898 goto jit_failure;
4899 }
4900 } else {
4901 break;
4902 }
4903 }
4904 if (orig_op1_type != IS_UNKNOWN
4905 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4906 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4907 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4908 goto jit_failure;
4909 }
4910 if (opline->op1_type == IS_CV
4911 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4912 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4913 }
4914 } else {
4915 CHECK_OP1_TRACE_TYPE();
4916 }
4917 op2_info = OP2_INFO();
4918 CHECK_OP2_TRACE_TYPE();
4919 op1_data_info = OP1_DATA_INFO();
4920 CHECK_OP1_DATA_TRACE_TYPE();
4921 if (!zend_jit_assign_dim(&dasm_state, opline,
4922 op1_info, op1_addr, op2_info, op1_data_info, val_type,
4923 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4924 goto jit_failure;
4925 }
4926 if ((opline+1)->op1_type == IS_CV
4927 && (ssa_op+1)->op1_def >= 0
4928 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4929 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4930 }
4931 goto done;
4932 case ZEND_ASSIGN:
4933 if (opline->op1_type != IS_CV) {
4934 break;
4935 }
4936 op2_addr = OP2_REG_ADDR();
4937 op2_info = OP2_INFO();
4938
4939 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
4940 op2_def_addr = op2_addr;
4941 } else {
4942 op2_def_addr = OP2_DEF_REG_ADDR();
4943 }
4944 CHECK_OP2_TRACE_TYPE();
4945 op1_info = OP1_INFO();
4946 op1_def_info = OP1_DEF_INFO();
4947 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
4948 if (op1_type < IS_STRING
4949 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
4950 if (!zend_jit_scalar_type_guard(&dasm_state, opline, opline->op1.var)) {
4951 goto jit_failure;
4952 }
4953 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
4954 } else {
4955 CHECK_OP1_TRACE_TYPE();
4956 }
4957 }
4958 op1_addr = OP1_REG_ADDR();
4959 op1_def_addr = OP1_DEF_REG_ADDR();
4960 if (Z_MODE(op1_def_addr) != IS_REG &&
4961 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
4962 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
4963 /* type may be not set */
4964 op1_info |= MAY_BE_NULL;
4965 }
4966 if (orig_op1_type != IS_UNKNOWN) {
4967 if (orig_op1_type & IS_TRACE_REFERENCE) {
4968 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4969 !ssa->var_info[ssa_op->op1_use].guarded_reference, 0)) {
4970 goto jit_failure;
4971 }
4972 if (opline->op1_type == IS_CV
4973 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4974 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4975 }
4976 if (opline->result_type == IS_UNUSED) {
4977 res_addr = 0;
4978 } else {
4979 res_addr = RES_REG_ADDR();
4980 if (Z_MODE(res_addr) != IS_REG
4981 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4982 send_result = 1;
4983 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4984 if (!zend_jit_reuse_ip(&dasm_state)) {
4985 goto jit_failure;
4986 }
4987 }
4988 }
4989 if (!zend_jit_assign_to_typed_ref(&dasm_state, opline, opline->op2_type, op2_addr, res_addr, 1)) {
4990 goto jit_failure;
4991 }
4992 op1_def_addr = op1_addr;
4993 op1_def_info &= ~MAY_BE_REF;
4994 } else if (op1_info & MAY_BE_REF) {
4995 if (!zend_jit_noref_guard(&dasm_state, opline, op1_addr)) {
4996 goto jit_failure;
4997 }
4998 op1_info &= ~MAY_BE_REF;
4999 op1_def_info &= ~MAY_BE_REF;
5000 }
5001 }
5002 if (opline->result_type == IS_UNUSED) {
5003 res_addr = 0;
5004 res_info = -1;
5005 } else {
5006 res_addr = RES_REG_ADDR();
5007 res_info = RES_INFO();
5008 if (Z_MODE(res_addr) != IS_REG
5009 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5010 send_result = 1;
5011 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5012 if (!zend_jit_reuse_ip(&dasm_state)) {
5013 goto jit_failure;
5014 }
5015 }
5016 }
5017 if (!zend_jit_assign(&dasm_state, opline,
5018 op1_info, op1_addr,
5019 op1_def_info, op1_def_addr,
5020 op2_info, op2_addr, op2_def_addr,
5021 res_info, res_addr,
5022 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5023 goto jit_failure;
5024 }
5025 if (ssa_op->op2_def > 0
5026 && Z_MODE(op2_addr) == IS_REG
5027 && ssa->vars[ssa_op->op2_def].no_val) {
5028 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5029 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5030
5031 if (STACK_MEM_TYPE(stack, var_num) != type
5032 && ssa->vars[ssa_op->op2_def].use_chain < 0
5033 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5034 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5035 return 0;
5036 }
5037 SET_STACK_TYPE(stack, var_num, type, 1);
5038 }
5039 }
5040 if (opline->op2_type == IS_CV
5041 && ssa_op->op2_def >= 0
5042 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5043 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5044 }
5045 goto done;
5046 case ZEND_CAST:
5047 if (opline->extended_value != op1_type) {
5048 break;
5049 }
5050 ZEND_FALLTHROUGH;
5051 case ZEND_QM_ASSIGN:
5052 op1_addr = OP1_REG_ADDR();
5053 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5054 op1_def_addr = op1_addr;
5055 } else {
5056 op1_def_addr = OP1_DEF_REG_ADDR();
5057 }
5058 op1_info = OP1_INFO();
5059 CHECK_OP1_TRACE_TYPE();
5060 res_info = RES_INFO();
5061 res_use_info = zend_jit_trace_type_to_info(
5062 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
5063 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
5064 res_addr = RES_REG_ADDR();
5065 if (Z_MODE(res_addr) != IS_REG &&
5066 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5067 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5068 /* type may be not set */
5069 res_use_info |= MAY_BE_NULL;
5070 }
5071 if (!zend_jit_qm_assign(&dasm_state, opline,
5072 op1_info, op1_addr, op1_def_addr,
5073 res_use_info, res_info, res_addr)) {
5074 goto jit_failure;
5075 }
5076 if (ssa_op->op1_def > 0
5077 && Z_MODE(op1_addr) == IS_REG
5078 && ssa->vars[ssa_op->op1_def].no_val) {
5079 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5080 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5081
5082 if (STACK_MEM_TYPE(stack, var_num) != type
5083 && ssa->vars[ssa_op->op1_def].use_chain < 0
5084 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5085 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5086 return 0;
5087 }
5088 SET_STACK_TYPE(stack, var_num, type, 1);
5089 }
5090 }
5091 if (opline->op1_type == IS_CV
5092 && ssa_op->op1_def >= 0
5093 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5094 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5095 }
5096 goto done;
5097 case ZEND_INIT_FCALL:
5098 case ZEND_INIT_FCALL_BY_NAME:
5099 case ZEND_INIT_NS_FCALL_BY_NAME:
5100 frame_flags = TRACE_FRAME_MASK_NESTED;
5101 if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5102 goto jit_failure;
5103 }
5104 goto done;
5105 case ZEND_SEND_VAL:
5106 case ZEND_SEND_VAL_EX:
5107 if (opline->op2_type == IS_CONST) {
5108 /* Named parameters not supported in JIT */
5109 break;
5110 }
5111 if (opline->opcode == ZEND_SEND_VAL_EX
5112 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5113 break;
5114 }
5115 op1_info = OP1_INFO();
5116 CHECK_OP1_TRACE_TYPE();
5117 if (!zend_jit_send_val(&dasm_state, opline,
5118 op1_info, OP1_REG_ADDR())) {
5119 goto jit_failure;
5120 }
5121 if (frame->call && frame->call->func) {
5122 if (opline->op1_type == IS_CONST) {
5123 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5124 } else if (op1_type != IS_UNKNOWN) {
5125 if (op1_type == IS_UNDEF) {
5126 op1_type = IS_NULL;
5127 }
5128 zend_jit_trace_send_type(opline, frame->call, op1_type);
5129 }
5130 }
5131 goto done;
5132 case ZEND_SEND_REF:
5133 if (opline->op2_type == IS_CONST) {
5134 /* Named parameters not supported in JIT */
5135 break;
5136 }
5137 op1_info = OP1_INFO();
5138 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
5139 op1_info, 0)) {
5140 goto jit_failure;
5141 }
5142 if (opline->op1_type == IS_CV
5143 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5144 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5145 }
5146 goto done;
5147 case ZEND_SEND_VAR:
5148 case ZEND_SEND_VAR_EX:
5149 case ZEND_SEND_VAR_NO_REF:
5150 case ZEND_SEND_VAR_NO_REF_EX:
5151 case ZEND_SEND_FUNC_ARG:
5152 if (opline->op2_type == IS_CONST) {
5153 /* Named parameters not supported in JIT */
5154 break;
5155 }
5156 if ((opline->opcode == ZEND_SEND_VAR_EX
5157 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5158 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5159 break;
5160 }
5161 op1_addr = OP1_REG_ADDR();
5162 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5163 op1_def_addr = op1_addr;
5164 } else {
5165 op1_def_addr = OP1_DEF_REG_ADDR();
5166 }
5167 op1_info = OP1_INFO();
5168 CHECK_OP1_TRACE_TYPE();
5169 if (!zend_jit_send_var(&dasm_state, opline, op_array,
5170 op1_info, op1_addr, op1_def_addr)) {
5171 goto jit_failure;
5172 }
5173 if (ssa_op->op1_def > 0
5174 && Z_MODE(op1_addr) == IS_REG
5175 && ssa->vars[ssa_op->op1_def].no_val) {
5176 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5177 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5178
5179 if (STACK_MEM_TYPE(stack, var_num) != type
5180 && ssa->vars[ssa_op->op1_def].use_chain < 0
5181 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5182 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5183 return 0;
5184 }
5185 SET_STACK_TYPE(stack, var_num, type, 1);
5186 }
5187 }
5188 if (opline->op1_type == IS_CV
5189 && ssa_op->op1_def >= 0
5190 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5191 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5192 }
5193 if (frame->call && frame->call->func) {
5194 if ((opline->opcode == ZEND_SEND_VAR_EX
5195 || opline->opcode == ZEND_SEND_FUNC_ARG)
5196 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5197 goto done;
5198 }
5199 if (op1_type != IS_UNKNOWN) {
5200 if (op1_type == IS_UNDEF) {
5201 op1_type = IS_NULL;
5202 }
5203 zend_jit_trace_send_type(opline, frame->call, op1_type);
5204 }
5205 }
5206 goto done;
5207 case ZEND_CHECK_FUNC_ARG:
5208 if (!JIT_G(current_frame)
5209 || !JIT_G(current_frame)->call
5210 || !JIT_G(current_frame)->call->func) {
5211 break;
5212 }
5213 if (opline->op2_type == IS_CONST
5214 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5215 /* Named parameters not supported in JIT */
5216 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5217 break;
5218 }
5219 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
5220 goto jit_failure;
5221 }
5222 goto done;
5223 case ZEND_CHECK_UNDEF_ARGS:
5224 if (JIT_G(current_frame)
5225 && JIT_G(current_frame)->call) {
5226 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5227 }
5228 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
5229 goto jit_failure;
5230 }
5231 goto done;
5232 case ZEND_DO_UCALL:
5233 case ZEND_DO_ICALL:
5234 case ZEND_DO_FCALL_BY_NAME:
5235 case ZEND_DO_FCALL:
5236 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5237 goto jit_failure;
5238 }
5239 goto done;
5240 case ZEND_IS_EQUAL:
5241 case ZEND_IS_NOT_EQUAL:
5242 case ZEND_IS_SMALLER:
5243 case ZEND_IS_SMALLER_OR_EQUAL:
5244 case ZEND_CASE:
5245 op1_info = OP1_INFO();
5246 op2_info = OP2_INFO();
5247 skip_comparison =
5248 ssa_op != ssa->ops &&
5249 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5250 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5251 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5252 CHECK_OP1_TRACE_TYPE();
5253 CHECK_OP2_TRACE_TYPE();
5254 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5255 bool exit_if_true = 0;
5256 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5257 uint32_t exit_point;
5258
5259 if (ra) {
5260 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5261 }
5262 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5263 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5264 if (!exit_addr) {
5265 goto jit_failure;
5266 }
5267 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5268 if (!zend_jit_cmp(&dasm_state, opline,
5269 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5270 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5271 RES_REG_ADDR(),
5272 zend_may_throw(opline, ssa_op, op_array, ssa),
5273 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5274 goto jit_failure;
5275 }
5276 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5277 } else {
5278 smart_branch_opcode = 0;
5279 exit_addr = NULL;
5280 if (!zend_jit_cmp(&dasm_state, opline,
5281 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5282 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5283 RES_REG_ADDR(),
5284 zend_may_throw(opline, ssa_op, op_array, ssa),
5285 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5286 goto jit_failure;
5287 }
5288 }
5289 goto done;
5290 case ZEND_IS_IDENTICAL:
5291 case ZEND_IS_NOT_IDENTICAL:
5292 case ZEND_CASE_STRICT:
5293 op1_info = OP1_INFO();
5294 op2_info = OP2_INFO();
5295 skip_comparison =
5296 ssa_op != ssa->ops &&
5297 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5298 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5299 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5300 CHECK_OP1_TRACE_TYPE();
5301 CHECK_OP2_TRACE_TYPE();
5302 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5303 bool exit_if_true = 0;
5304 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5305 uint32_t exit_point;
5306
5307 if (ra) {
5308 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5309 }
5310 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5311 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5312 if (!exit_addr) {
5313 goto jit_failure;
5314 }
5315 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5316 exit_if_true = !exit_if_true;
5317 }
5318 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5319 if (!zend_jit_identical(&dasm_state, opline,
5320 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5321 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5322 RES_REG_ADDR(),
5323 zend_may_throw(opline, ssa_op, op_array, ssa),
5324 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5325 goto jit_failure;
5326 }
5327 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5328 } else {
5329 smart_branch_opcode = 0;
5330 exit_addr = NULL;
5331 if (!zend_jit_identical(&dasm_state, opline,
5332 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5333 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5334 RES_REG_ADDR(),
5335 zend_may_throw(opline, ssa_op, op_array, ssa),
5336 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5337 goto jit_failure;
5338 }
5339 }
5340 goto done;
5341 case ZEND_DEFINED:
5342 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5343 bool exit_if_true = 0;
5344 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5345 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5346
5347 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5348 if (!exit_addr) {
5349 goto jit_failure;
5350 }
5351 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5352 } else {
5353 smart_branch_opcode = 0;
5354 exit_addr = NULL;
5355 }
5356 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5357 goto jit_failure;
5358 }
5359 goto done;
5360 case ZEND_TYPE_CHECK:
5361 if (opline->extended_value == MAY_BE_RESOURCE) {
5362 // TODO: support for is_resource() ???
5363 break;
5364 }
5365 op1_info = OP1_INFO();
5366 CHECK_OP1_TRACE_TYPE();
5367 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5368 bool exit_if_true = 0;
5369 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5370 uint32_t exit_point;
5371
5372 if (ra) {
5373 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5374 }
5375 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5376 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5377 if (!exit_addr) {
5378 goto jit_failure;
5379 }
5380 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5381 } else {
5382 smart_branch_opcode = 0;
5383 exit_addr = NULL;
5384 }
5385 if (!zend_jit_type_check(&dasm_state, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5386 goto jit_failure;
5387 }
5388 goto done;
5389 case ZEND_RETURN:
5390 op1_info = OP1_INFO();
5391 CHECK_OP1_TRACE_TYPE();
5392 if (opline->op1_type == IS_CONST) {
5393 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5394 } else if (op1_type != IS_UNKNOWN) {
5395 res_type = op1_type;
5396 }
5397 if (op_array->type == ZEND_EVAL_CODE
5398 // TODO: support for top-level code
5399 || !op_array->function_name
5400 // TODO: support for IS_UNDEF ???
5401 || (op1_info & MAY_BE_UNDEF)) {
5402 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5403 goto jit_failure;
5404 }
5405 } else {
5406 int j;
5407 int may_throw = 0;
5408 bool left_frame = 0;
5409
5410 if (!zend_jit_return(&dasm_state, opline, op_array,
5411 op1_info, OP1_REG_ADDR())) {
5412 goto jit_failure;
5413 }
5414 if (op_array->last_var > 100) {
5415 /* To many CVs to unroll */
5416 if (!zend_jit_free_cvs(&dasm_state)) {
5417 goto jit_failure;
5418 }
5419 left_frame = 1;
5420 }
5421 if (!left_frame) {
5422 for (j = 0 ; j < op_array->last_var; j++) {
5423 uint32_t info;
5424 zend_uchar type;
5425
5426 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5427 type = STACK_TYPE(stack, j);
5428 info = zend_jit_trace_type_to_info_ex(type, info);
5429 if (opline->op1_type == IS_CV
5430 && EX_VAR_TO_NUM(opline->op1.var) == j
5431 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5432 if (JIT_G(current_frame)
5433 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5434 continue;
5435 } else {
5436 info |= MAY_BE_NULL;
5437 }
5438 }
5439 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5440 if (!left_frame) {
5441 left_frame = 1;
5442 if (!zend_jit_leave_frame(&dasm_state)) {
5443 goto jit_failure;
5444 }
5445 }
5446 if (!zend_jit_free_cv(&dasm_state, info, j)) {
5447 goto jit_failure;
5448 }
5449 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5450 if (info & MAY_BE_RC1) {
5451 may_throw = 1;
5452 }
5453 }
5454 }
5455 }
5456 }
5457 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
5458 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5459 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5460 goto jit_failure;
5461 }
5462 }
5463 goto done;
5464 case ZEND_BOOL:
5465 case ZEND_BOOL_NOT:
5466 op1_info = OP1_INFO();
5467 CHECK_OP1_TRACE_TYPE();
5468 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5469 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5470 -1, -1,
5471 zend_may_throw(opline, ssa_op, op_array, ssa),
5472 opline->opcode, NULL)) {
5473 goto jit_failure;
5474 }
5475 goto done;
5476 case ZEND_JMPZ:
5477 case ZEND_JMPNZ:
5478 case ZEND_JMPZNZ:
5479 case ZEND_JMPZ_EX:
5480 case ZEND_JMPNZ_EX:
5481 op1_info = OP1_INFO();
5482 CHECK_OP1_TRACE_TYPE();
5483 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5484 const zend_op *exit_opline = NULL;
5485 uint32_t exit_point;
5486
5487 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5488 /* taken branch */
5489 if (opline->opcode == ZEND_JMPNZ_EX) {
5490 smart_branch_opcode = ZEND_JMPZ_EX;
5491 } else if (opline->opcode == ZEND_JMPZ_EX) {
5492 smart_branch_opcode = ZEND_JMPNZ_EX;
5493 } else if (opline->opcode == ZEND_JMPNZ) {
5494 smart_branch_opcode = ZEND_JMPZ;
5495 } else {
5496 smart_branch_opcode = ZEND_JMPNZ;
5497 }
5498 exit_opline = (opline->opcode == ZEND_JMPZNZ) ?
5499 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
5500 opline + 1;
5501 } else if (opline->opcode == ZEND_JMPZNZ) {
5502 ZEND_ASSERT((p+1)->opline == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value));
5503 smart_branch_opcode = ZEND_JMPZ;
5504 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5505 } else if ((p+1)->opline == opline + 1) {
5506 /* not taken branch */
5507 smart_branch_opcode = opline->opcode;
5508 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5509 } else {
5510 ZEND_UNREACHABLE();
5511 }
5512 if (ra) {
5513 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5514 }
5515 if (!(op1_info & MAY_BE_GUARD)
5516 && has_concrete_type(op1_info)
5517 && concrete_type(op1_info) <= IS_TRUE) {
5518 /* unconditional branch */
5519 exit_addr = NULL;
5520 } else if (opline->result_type == IS_TMP_VAR) {
5521 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5522 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5523
5524 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5525 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5526 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5527 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5528 if (!exit_addr) {
5529 goto jit_failure;
5530 }
5531 } else {
5532 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5533 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5534 if (!exit_addr) {
5535 goto jit_failure;
5536 }
5537 }
5538 } else {
5539 ZEND_UNREACHABLE();
5540 }
5541 if (opline->result_type == IS_UNDEF) {
5542 res_addr = 0;
5543 } else {
5544 res_addr = RES_REG_ADDR();
5545 }
5546 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5547 op1_info, OP1_REG_ADDR(), res_addr,
5548 -1, -1,
5549 zend_may_throw(opline, ssa_op, op_array, ssa),
5550 smart_branch_opcode, exit_addr)) {
5551 goto jit_failure;
5552 }
5553 goto done;
5554 case ZEND_ISSET_ISEMPTY_CV:
5555 if ((opline->extended_value & ZEND_ISEMPTY)) {
5556 // TODO: support for empty() ???
5557 break;
5558 }
5559 op1_info = OP1_INFO();
5560 op1_addr = OP1_REG_ADDR();
5561 if (orig_op1_type != IS_UNKNOWN
5562 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5563 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5564 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5565 goto jit_failure;
5566 }
5567 if (opline->op1_type == IS_CV
5568 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5569 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5570 }
5571 } else {
5572 CHECK_OP1_TRACE_TYPE();
5573 }
5574 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5575 bool exit_if_true = 0;
5576 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5577 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5578
5579 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5580 if (!exit_addr) {
5581 goto jit_failure;
5582 }
5583 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5584 } else {
5585 smart_branch_opcode = 0;
5586 exit_addr = NULL;
5587 }
5588 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
5589 op1_info, op1_addr,
5590 smart_branch_opcode, -1, -1, exit_addr)) {
5591 goto jit_failure;
5592 }
5593 goto done;
5594 case ZEND_IN_ARRAY:
5595 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5596 break;
5597 }
5598 op1_info = OP1_INFO();
5599 op1_addr = OP1_REG_ADDR();
5600 CHECK_OP1_TRACE_TYPE();
5601 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5602 break;
5603 }
5604 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5605 bool exit_if_true = 0;
5606 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5607 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5608
5609 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5610 if (!exit_addr) {
5611 goto jit_failure;
5612 }
5613 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5614 } else {
5615 smart_branch_opcode = 0;
5616 exit_addr = NULL;
5617 }
5618 if (!zend_jit_in_array(&dasm_state, opline,
5619 op1_info, op1_addr,
5620 smart_branch_opcode, -1, -1, exit_addr)) {
5621 goto jit_failure;
5622 }
5623 goto done;
5624 case ZEND_FETCH_DIM_FUNC_ARG:
5625 if (!JIT_G(current_frame)
5626 || !JIT_G(current_frame)->call
5627 || !JIT_G(current_frame)->call->func
5628 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5629 break;
5630 }
5631 ZEND_FALLTHROUGH;
5632 case ZEND_FETCH_DIM_R:
5633 case ZEND_FETCH_DIM_IS:
5634 case ZEND_FETCH_LIST_R:
5635 op1_info = OP1_INFO();
5636 op1_addr = OP1_REG_ADDR();
5637 if (orig_op1_type != IS_UNKNOWN
5638 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5639 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5640 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5641 goto jit_failure;
5642 }
5643 if (opline->op1_type == IS_CV
5644 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5645 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5646 if (ssa_op->op1_def >= 0) {
5647 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5648 }
5649 }
5650 } else {
5651 CHECK_OP1_TRACE_TYPE();
5652 }
5653 op2_info = OP2_INFO();
5654 CHECK_OP2_TRACE_TYPE();
5655 res_info = RES_INFO();
5656 avoid_refcounting =
5657 ssa_op->op1_use >= 0 &&
5658 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5659 if (op1_info & MAY_BE_PACKED_GUARD) {
5660 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5661 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5662 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5663 && MAY_BE_PACKED(op1_info)
5664 && MAY_BE_HASH(op1_info)
5665 && orig_op1_type != IS_UNKNOWN) {
5666 op1_info |= MAY_BE_PACKED_GUARD;
5667 if (orig_op1_type & IS_TRACE_PACKED) {
5668 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5669 if (op1_type != IS_UNKNOWN) {
5670 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5671 }
5672 } else {
5673 op1_info &= ~MAY_BE_ARRAY_PACKED;
5674 if (op1_type != IS_UNKNOWN) {
5675 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5676 }
5677 }
5678 }
5679 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
5680 op1_info, op1_addr, avoid_refcounting,
5681 op2_info, res_info, RES_REG_ADDR(), val_type)) {
5682 goto jit_failure;
5683 }
5684 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5685 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5686 }
5687 goto done;
5688 case ZEND_FETCH_DIM_W:
5689 case ZEND_FETCH_DIM_RW:
5690 // case ZEND_FETCH_DIM_UNSET:
5691 case ZEND_FETCH_LIST_W:
5692 if (opline->op1_type != IS_CV
5693 && (orig_op1_type == IS_UNKNOWN
5694 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5695 break;
5696 }
5697 op1_info = OP1_INFO();
5698 op1_addr = OP1_REG_ADDR();
5699 if (opline->op1_type == IS_VAR) {
5700 if (orig_op1_type != IS_UNKNOWN
5701 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5702 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5703 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5704 goto jit_failure;
5705 }
5706 } else {
5707 break;
5708 }
5709 }
5710 if (orig_op1_type != IS_UNKNOWN
5711 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5712 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5713 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5714 goto jit_failure;
5715 }
5716 if (opline->op1_type == IS_CV
5717 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5718 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5719 }
5720 } else {
5721 CHECK_OP1_TRACE_TYPE();
5722 }
5723 op2_info = OP2_INFO();
5724 CHECK_OP2_TRACE_TYPE();
5725 op1_def_info = OP1_DEF_INFO();
5726 if (!zend_jit_fetch_dim(&dasm_state, opline,
5727 op1_info, op1_addr, op2_info, RES_REG_ADDR(), val_type)) {
5728 goto jit_failure;
5729 }
5730 if (ssa_op->result_def > 0
5731 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5732 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5733 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5734 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5735 }
5736 goto done;
5737 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5738 if ((opline->extended_value & ZEND_ISEMPTY)) {
5739 // TODO: support for empty() ???
5740 break;
5741 }
5742 op1_info = OP1_INFO();
5743 op1_addr = OP1_REG_ADDR();
5744 if (orig_op1_type != IS_UNKNOWN
5745 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5746 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5747 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5748 goto jit_failure;
5749 }
5750 if (opline->op1_type == IS_CV
5751 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5752 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5753 }
5754 } else {
5755 CHECK_OP1_TRACE_TYPE();
5756 }
5757 op2_info = OP2_INFO();
5758 CHECK_OP2_TRACE_TYPE();
5759 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5760 bool exit_if_true = 0;
5761 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5762 uint32_t exit_point;
5763
5764 if (ra) {
5765 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5766 }
5767 if (ssa_op->op1_use >= 0
5768 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5769 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5770 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5771 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5772
5773 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5774 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5775 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5776 } else {
5777 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5778 }
5779 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5780 if (!exit_addr) {
5781 goto jit_failure;
5782 }
5783 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5784 } else {
5785 smart_branch_opcode = 0;
5786 exit_addr = NULL;
5787 }
5788 avoid_refcounting =
5789 ssa_op->op1_use >= 0 &&
5790 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5791 if (op1_info & MAY_BE_PACKED_GUARD) {
5792 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5793 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5794 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5795 && MAY_BE_PACKED(op1_info)
5796 && MAY_BE_HASH(op1_info)
5797 && orig_op1_type != IS_UNKNOWN) {
5798 op1_info |= MAY_BE_PACKED_GUARD;
5799 if (orig_op1_type & IS_TRACE_PACKED) {
5800 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5801 } else {
5802 op1_info &= ~MAY_BE_ARRAY_PACKED;
5803 }
5804 }
5805 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
5806 op1_info, op1_addr, avoid_refcounting,
5807 op2_info, val_type,
5808 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5809 smart_branch_opcode, -1, -1,
5810 exit_addr)) {
5811 goto jit_failure;
5812 }
5813 goto done;
5814 case ZEND_FETCH_OBJ_FUNC_ARG:
5815 if (!JIT_G(current_frame)
5816 || !JIT_G(current_frame)->call
5817 || !JIT_G(current_frame)->call->func
5818 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5819 break;
5820 }
5821 ZEND_FALLTHROUGH;
5822 case ZEND_FETCH_OBJ_R:
5823 case ZEND_FETCH_OBJ_IS:
5824 case ZEND_FETCH_OBJ_W:
5825 on_this = delayed_fetch_this = 0;
5826 avoid_refcounting = 0;
5827 if (opline->op2_type != IS_CONST
5828 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5829 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5830 break;
5831 }
5832 ce = NULL;
5833 ce_is_instanceof = 0;
5834 op1_indirect = 0;
5835 if (opline->op1_type == IS_UNUSED) {
5836 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
5837 ce = op_array->scope;
5838 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
5839 op1_addr = 0;
5840 on_this = 1;
5841 } else {
5842 op1_info = OP1_INFO();
5843 if (!(op1_info & MAY_BE_OBJECT)) {
5844 break;
5845 }
5846 op1_addr = OP1_REG_ADDR();
5847 if (opline->op1_type == IS_VAR
5848 && opline->opcode == ZEND_FETCH_OBJ_W) {
5849 if (orig_op1_type != IS_UNKNOWN
5850 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5851 op1_indirect = 1;
5852 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5853 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5854 goto jit_failure;
5855 }
5856 }
5857 }
5858 if (orig_op1_type != IS_UNKNOWN
5859 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5860 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5861 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5862 goto jit_failure;
5863 }
5864 if (opline->op1_type == IS_CV
5865 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5866 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
5867 }
5868 } else {
5869 CHECK_OP1_TRACE_TYPE();
5870 }
5871 if (!(op1_info & MAY_BE_OBJECT)) {
5872 break;
5873 }
5874 if (ssa->var_info && ssa->ops) {
5875 if (ssa_op->op1_use >= 0) {
5876 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5877 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5878 ce = op1_ssa->ce;
5879 ce_is_instanceof = op1_ssa->is_instanceof;
5880 }
5881 }
5882 }
5883 if (ssa_op->op1_use >= 0) {
5884 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
5885 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5886 }
5887 if (delayed_fetch_this) {
5888 on_this = 1;
5889 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
5890 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
5891 } else if (op_array_ssa->ops
5892 && op_array_ssa->vars
5893 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
5894 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
5895 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
5896 }
5897 }
5898 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
5899 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
5900 on_this, delayed_fetch_this, avoid_refcounting, op1_ce, val_type,
5901 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
5902 goto jit_failure;
5903 }
5904 goto done;
5905 case ZEND_BIND_GLOBAL:
5906 orig_opline = opline;
5907 orig_ssa_op = ssa_op;
5908 while (1) {
5909 if (!ssa->ops || !ssa->var_info) {
5910 op1_info = MAY_BE_ANY|MAY_BE_REF;
5911 } else {
5912 op1_info = OP1_INFO();
5913 }
5914 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5915 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5916 }
5917 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
5918 goto jit_failure;
5919 }
5920 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
5921 opline++;
5922 ssa_op++;
5923 } else {
5924 break;
5925 }
5926 }
5927 opline = orig_opline;
5928 ssa_op = orig_ssa_op;
5929 goto done;
5930 case ZEND_RECV:
5931 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
5932 goto jit_failure;
5933 }
5934 goto done;
5935 case ZEND_RECV_INIT:
5936 orig_opline = opline;
5937 orig_ssa_op = ssa_op;
5938 while (1) {
5939 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
5940 (opline + 1)->opcode != ZEND_RECV_INIT,
5941 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5942 goto jit_failure;
5943 }
5944 if ((opline+1)->opcode == ZEND_RECV_INIT) {
5945 opline++;
5946 ssa_op++;
5947 } else {
5948 break;
5949 }
5950 }
5951 opline = orig_opline;
5952 ssa_op = orig_ssa_op;
5953 goto done;
5954 case ZEND_FREE:
5955 case ZEND_FE_FREE:
5956 op1_info = OP1_INFO();
5957 if (!zend_jit_free(&dasm_state, opline, op1_info,
5958 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5959 goto jit_failure;
5960 }
5961 goto done;
5962 case ZEND_ECHO:
5963 op1_info = OP1_INFO();
5964 CHECK_OP1_TRACE_TYPE();
5965 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5966 break;
5967 }
5968 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
5969 goto jit_failure;
5970 }
5971 goto done;
5972 case ZEND_STRLEN:
5973 op1_info = OP1_INFO();
5974 op1_addr = OP1_REG_ADDR();
5975 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
5976 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5977 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5978 goto jit_failure;
5979 }
5980 if (opline->op1_type == IS_CV
5981 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5982 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5983 }
5984 } else {
5985 CHECK_OP1_TRACE_TYPE();
5986 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5987 break;
5988 }
5989 }
5990 if (!zend_jit_strlen(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR())) {
5991 goto jit_failure;
5992 }
5993 goto done;
5994 case ZEND_COUNT:
5995 op1_info = OP1_INFO();
5996 op1_addr = OP1_REG_ADDR();
5997 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
5998 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5999 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6000 goto jit_failure;
6001 }
6002 if (opline->op1_type == IS_CV
6003 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6004 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6005 }
6006 } else {
6007 CHECK_OP1_TRACE_TYPE();
6008 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6009 break;
6010 }
6011 }
6012 if (!zend_jit_count(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6013 goto jit_failure;
6014 }
6015 goto done;
6016 case ZEND_FETCH_THIS:
6017 delayed_fetch_this = 0;
6018 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6019 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6020 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6021 delayed_fetch_this = 1;
6022 }
6023 }
6024 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) {
6025 goto jit_failure;
6026 }
6027 goto done;
6028 case ZEND_SWITCH_LONG:
6029 case ZEND_SWITCH_STRING:
6030 case ZEND_MATCH:
6031 if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6032 goto jit_failure;
6033 }
6034 goto done;
6035 case ZEND_VERIFY_RETURN_TYPE:
6036 if (opline->op1_type == IS_UNUSED) {
6037 /* Always throws */
6038 break;
6039 }
6040 if (opline->op1_type == IS_CONST) {
6041 /* TODO Different instruction format, has return value */
6042 break;
6043 }
6044 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6045 /* Not worth bothering with */
6046 break;
6047 }
6048 op1_info = OP1_INFO();
6049 CHECK_OP1_TRACE_TYPE();
6050 if (op1_info & MAY_BE_REF) {
6051 /* TODO May need reference unwrapping. */
6052 break;
6053 }
6054 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, op1_info)) {
6055 goto jit_failure;
6056 }
6057 goto done;
6058 case ZEND_FE_RESET_R:
6059 op1_info = OP1_INFO();
6060 CHECK_OP1_TRACE_TYPE();
6061 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6062 break;
6063 }
6064 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
6065 goto jit_failure;
6066 }
6067 goto done;
6068 case ZEND_FE_FETCH_R:
6069 op1_info = OP1_INFO();
6070 CHECK_OP1_TRACE_TYPE();
6071 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6072 break;
6073 }
6074 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6075 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6076 uint32_t exit_point;
6077
6078 if ((p+1)->opline == exit_opline) {
6079 /* taken branch (exit from loop) */
6080 exit_opline = opline;
6081 smart_branch_opcode = ZEND_NOP;
6082 } else if ((p+1)->opline == opline + 1) {
6083 /* not taken branch (loop) */
6084 smart_branch_opcode = ZEND_JMP;
6085 } else {
6086 ZEND_UNREACHABLE();
6087 }
6088 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6089 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6090 if (!exit_addr) {
6091 goto jit_failure;
6092 }
6093 } else {
6094 ZEND_UNREACHABLE();
6095 }
6096 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
6097 -1, smart_branch_opcode, exit_addr)) {
6098 goto jit_failure;
6099 }
6100 goto done;
6101 case ZEND_FETCH_CONSTANT:
6102 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6103 goto jit_failure;
6104 }
6105 goto done;
6106 case ZEND_INIT_METHOD_CALL:
6107 if (opline->op2_type != IS_CONST
6108 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6109 goto generic_dynamic_call;
6110 }
6111 on_this = delayed_fetch_this = 0;
6112 ce = NULL;
6113 ce_is_instanceof = 0;
6114 if (opline->op1_type == IS_UNUSED) {
6115 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6116 ce = op_array->scope;
6117 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
6118 op1_addr = 0;
6119 on_this = 1;
6120 } else {
6121 op1_info = OP1_INFO();
6122 op1_addr = OP1_REG_ADDR();
6123 if (polymorphic_side_trace) {
6124 op1_info = MAY_BE_OBJECT;
6125 op1_addr = 0;
6126 } else if (orig_op1_type != IS_UNKNOWN
6127 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6128 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6129 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6130 goto jit_failure;
6131 }
6132 if (opline->op1_type == IS_CV
6133 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6134 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6135 }
6136 } else {
6137 CHECK_OP1_TRACE_TYPE();
6138 }
6139 if (ssa->var_info && ssa->ops) {
6140 if (ssa_op->op1_use >= 0) {
6141 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6142 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6143 ce = op1_ssa->ce;
6144 ce_is_instanceof = op1_ssa->is_instanceof;
6145 }
6146 }
6147 }
6148 if (ssa_op->op1_use >= 0) {
6149 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6150 }
6151 if (delayed_fetch_this) {
6152 on_this = 1;
6153 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6154 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6155 } else if (op_array_ssa->ops
6156 && op_array_ssa->vars
6157 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6158 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6159 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6160 }
6161 }
6162 frame_flags = TRACE_FRAME_MASK_NESTED;
6163 if (!zend_jit_init_method_call(&dasm_state, opline,
6164 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6165 op_array, ssa, ssa_op, frame->call_level,
6166 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6167 p + 1, peek_checked_stack - checked_stack, polymorphic_side_trace)) {
6168 goto jit_failure;
6169 }
6170 goto done;
6171 case ZEND_INIT_DYNAMIC_CALL:
6172 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6173 goto generic_dynamic_call;
6174 }
6175 op2_info = OP2_INFO();
6176 CHECK_OP2_TRACE_TYPE();
6177 frame_flags = TRACE_FRAME_MASK_NESTED;
6178 if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6179 goto jit_failure;
6180 }
6181 goto done;
6182 case ZEND_INIT_STATIC_METHOD_CALL:
6183 generic_dynamic_call:
6184 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
6185 goto jit_failure;
6186 }
6187 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func
6188 && (opline->opcode != ZEND_INIT_STATIC_METHOD_CALL
6189 || opline->op1_type != IS_CONST
6190 || opline->op2_type != IS_CONST
6191 || zend_jit_may_be_modified((p+1)->func, op_array))) {
6192 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
6193 goto jit_failure;
6194 }
6195 }
6196 goto done;
6197 case ZEND_INIT_USER_CALL:
6198 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
6199 goto jit_failure;
6200 }
6201 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func
6202 && (opline->op2_type != IS_CONST
6203 || zend_jit_may_be_modified((p+1)->func, op_array))) {
6204 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
6205 goto jit_failure;
6206 }
6207 }
6208 goto done;
6209 case ZEND_NEW:
6210 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
6211 goto jit_failure;
6212 }
6213 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func
6214 && (opline->op1_type != IS_CONST
6215 || zend_jit_may_be_modified((p+1)->func, op_array))) {
6216 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6217 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
6218 goto jit_failure;
6219 }
6220 }
6221 goto done;
6222 case ZEND_SEND_ARRAY:
6223 case ZEND_SEND_UNPACK:
6224 if (JIT_G(current_frame)
6225 && JIT_G(current_frame)->call) {
6226 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6227 }
6228 break;
6229 case ZEND_ROPE_INIT:
6230 case ZEND_ROPE_ADD:
6231 case ZEND_ROPE_END:
6232 op2_info = OP2_INFO();
6233 CHECK_OP2_TRACE_TYPE();
6234 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6235 break;
6236 }
6237 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
6238 goto jit_failure;
6239 }
6240 goto done;
6241 default:
6242 break;
6243 }
6244 }
6245
6246 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6247 gen_handler = 1;
6248 op1_info = OP1_INFO();
6249 op2_info = OP2_INFO();
6250 if (op1_info & MAY_BE_GUARD) {
6251 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6252 }
6253 if (op2_info & MAY_BE_GUARD) {
6254 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6255 }
6256 if (!zend_jit_trace_handler(&dasm_state, op_array, opline,
6257 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6258 goto jit_failure;
6259 }
6260 }
6261
6262 done:
6263 polymorphic_side_trace = 0;
6264 switch (opline->opcode) {
6265 case ZEND_DO_FCALL:
6266 case ZEND_DO_ICALL:
6267 case ZEND_DO_UCALL:
6268 case ZEND_DO_FCALL_BY_NAME:
6269 case ZEND_CALLABLE_CONVERT:
6270 frame->call_level--;
6271 }
6272
6273 if (ra) {
6274 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
6275 }
6276
6277 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6278 && STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var)) > ZREG_NUM) {
6279 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6280 }
6281
6282 if (opline->opcode == ZEND_ROPE_INIT) {
6283 /* clear stack slots used by rope */
6284 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6285 uint32_t count =
6286 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6287
6288 do {
6289 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6290 var++;
6291 count--;
6292 } while (count);
6293 }
6294
6295 if (ssa_op) {
6296 zend_ssa_range tmp;
6297
6298 /* Keep information about known types on abstract stack */
6299 if (ssa_op->result_def >= 0) {
6300 zend_uchar type = IS_UNKNOWN;
6301
6302 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6303 || send_result) {
6304 /* we didn't set result variable */
6305 type = IS_UNKNOWN;
6306 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6307 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6308 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6309 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6310 if (opline->op1_type != IS_CONST) {
6311 /* copy */
6312 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6313 }
6314 } else if (opline->opcode == ZEND_ASSIGN) {
6315 if (opline->op2_type != IS_CONST
6316 && ssa_op->op1_use >= 0
6317 /* assignment to typed reference may cause conversion */
6318 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6319 /* copy */
6320 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6321 }
6322 } else if (opline->opcode == ZEND_POST_INC
6323 || opline->opcode == ZEND_POST_DEC) {
6324 /* copy */
6325 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6326 }
6327 if (opline->opcode == ZEND_JMP_SET
6328 || opline->opcode == ZEND_COALESCE
6329 || opline->opcode == ZEND_JMP_NULL) {
6330 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6331 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6332 } else if ((p+1)->opline != (opline + 1)) {
6333 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6334 }
6335 } else {
6336 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6337 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->result_def]));
6338 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6339 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6340 }
6341 if (type != IS_UNKNOWN) {
6342 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6343 if (opline->opcode == ZEND_FETCH_THIS
6344 && delayed_fetch_this) {
6345 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_THIS);
6346 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6347 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_TRY_ADDREF);
6348 } else if (ra && ra[ssa_op->result_def]) {
6349 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6350 ra[ssa_op->result_def]->flags & ZREG_STORE);
6351 }
6352 }
6353 }
6354
6355 if (type == IS_LONG
6356 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->result_def, &tmp)) {
6357 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6358 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6359 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6360 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6361 ssa->var_info[ssa_op->result_def].has_range = 1;
6362 }
6363 }
6364 if (ssa_op->op1_def >= 0
6365 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6366 || opline->result_type != IS_CV
6367 || opline->result.var != opline->op1.var)) {
6368 zend_uchar type = IS_UNKNOWN;
6369
6370 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6371 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6372 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6373 } else if (opline->opcode == ZEND_ASSIGN) {
6374 if (!(OP1_INFO() & MAY_BE_REF)
6375 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6376 if (opline->op2_type != IS_CONST) {
6377 /* copy */
6378 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6379 }
6380 }
6381 } else if (opline->opcode == ZEND_SEND_VAR
6382 || opline->opcode == ZEND_CAST
6383 || opline->opcode == ZEND_QM_ASSIGN
6384 || opline->opcode == ZEND_JMP_SET
6385 || opline->opcode == ZEND_COALESCE
6386 || opline->opcode == ZEND_JMP_NULL
6387 || opline->opcode == ZEND_FE_RESET_R) {
6388 /* keep old value */
6389 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6390 }
6391 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6392 (gen_handler || type == IS_UNKNOWN || !ra ||
6393 (!ra[ssa_op->op1_def] &&
6394 !(ssa->vars[ssa_op->op1_def].no_val &&
6395 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6396 (opline->opcode == ZEND_QM_ASSIGN ||
6397 opline->opcode == ZEND_SEND_VAR ||
6398 opline->opcode == ZEND_SEND_VAR_EX ||
6399 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6400 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6401 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6402 if (type != IS_UNKNOWN) {
6403 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6404 if (ra && ra[ssa_op->op1_def]) {
6405 uint8_t flags = ra[ssa_op->op1_def]->flags & ZREG_STORE;
6406
6407 if (ssa_op->op1_use >= 0) {
6408 if (opline->opcode == ZEND_SEND_VAR
6409 || opline->opcode == ZEND_CAST
6410 || opline->opcode == ZEND_QM_ASSIGN
6411 || opline->opcode == ZEND_JMP_SET
6412 || opline->opcode == ZEND_COALESCE
6413 || opline->opcode == ZEND_JMP_NULL
6414 || opline->opcode == ZEND_FE_RESET_R) {
6415 if (!ra[ssa_op->op1_use]) {
6416 flags |= ZREG_LOAD;
6417 }
6418 }
6419 }
6420 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, flags);
6421 }
6422 }
6423 if (type == IS_LONG
6424 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op1_def, &tmp)) {
6425 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6426 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6427 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6428 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6429 ssa->var_info[ssa_op->op1_def].has_range = 1;
6430 }
6431 }
6432 if (ssa_op->op2_def >= 0
6433 && (opline->opcode != ZEND_ASSIGN
6434 || opline->op1_type != IS_CV
6435 || opline->op1.var != opline->op2.var)) {
6436 zend_uchar type = IS_UNKNOWN;
6437
6438 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6439 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6440 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6441 } else if (opline->opcode == ZEND_ASSIGN) {
6442 /* keep old value */
6443 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6444 }
6445 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6446 (gen_handler || type == IS_UNKNOWN || !ra ||
6447 (!ra[ssa_op->op2_def] &&
6448 !(ssa->vars[ssa_op->op2_def].no_val &&
6449 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6450 opline->opcode == ZEND_ASSIGN))));
6451 if (type != IS_UNKNOWN) {
6452 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6453 if (ra && ra[ssa_op->op2_def]) {
6454 uint8_t flags = ra[ssa_op->op2_def]->flags & ZREG_STORE;
6455
6456 if (ssa_op->op2_use >= 0) {
6457 if (opline->opcode == ZEND_ASSIGN) {
6458 if (!ra[ssa_op->op2_use]
6459 || ra[ssa_op->op2_use]->reg != ra[ssa_op->op2_def]->reg) {
6460 flags |= ZREG_LOAD;
6461 }
6462 }
6463 }
6464 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def]->reg, flags);
6465 }
6466 }
6467 if (type == IS_LONG
6468 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op2_def, &tmp)) {
6469 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6470 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6471 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6472 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6473 ssa->var_info[ssa_op->op2_def].has_range = 1;
6474 }
6475 }
6476
6477 switch (opline->opcode) {
6478 case ZEND_ASSIGN_DIM:
6479 case ZEND_ASSIGN_OBJ:
6480 case ZEND_ASSIGN_STATIC_PROP:
6481 case ZEND_ASSIGN_DIM_OP:
6482 case ZEND_ASSIGN_OBJ_OP:
6483 case ZEND_ASSIGN_STATIC_PROP_OP:
6484 case ZEND_ASSIGN_OBJ_REF:
6485 case ZEND_ASSIGN_STATIC_PROP_REF:
6486 /* OP_DATA */
6487 ssa_op++;
6488 opline++;
6489 if (ssa_op->op1_def >= 0) {
6490 zend_uchar type = IS_UNKNOWN;
6491
6492 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6493 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6494 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6495 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6496 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6497 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6498 /* keep old value */
6499 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6500 }
6501 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6502 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->op1_def]));
6503 if (type != IS_UNKNOWN) {
6504 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6505 if (ra && ra[ssa_op->op1_def]) {
6506 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6507 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6508 }
6509 }
6510 if (type == IS_LONG
6511 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op1_def, &tmp)) {
6512 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6513 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6514 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6515 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6516 ssa->var_info[ssa_op->op1_def].has_range = 1;
6517 }
6518 }
6519 ssa_op++;
6520 break;
6521 case ZEND_RECV_INIT:
6522 ssa_op++;
6523 opline++;
6524 while (opline->opcode == ZEND_RECV_INIT) {
6525 if (ssa_op->result_def >= 0) {
6526 zend_uchar type = IS_UNKNOWN;
6527
6528 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6529 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6530 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6531 }
6532 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6533 (gen_handler || !ra || !ra[ssa_op->result_def]));
6534 if (ra && ra[ssa_op->result_def]) {
6535 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6536 ra[ssa_op->result_def]->flags & ZREG_STORE);
6537 }
6538 }
6539 ssa_op++;
6540 opline++;
6541 }
6542 break;
6543 case ZEND_BIND_GLOBAL:
6544 ssa_op++;
6545 opline++;
6546 while (opline->opcode == ZEND_BIND_GLOBAL) {
6547 if (ssa_op->op1_def >= 0) {
6548 zend_uchar type = IS_UNKNOWN;
6549
6550 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6551 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6552 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6553 }
6554 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6555 (gen_handler || !ra || !ra[ssa_op->op1_def]));
6556 if (ra && ra[ssa_op->op1_def]) {
6557 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6558 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6559 }
6560 }
6561 ssa_op++;
6562 opline++;
6563 }
6564 break;
6565 default:
6566 ssa_op += zend_jit_trace_op_len(opline);
6567 break;
6568 }
6569
6570 if (send_result) {
6571 ssa_op++;
6572 p++;
6573 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6574 p++;
6575 }
6576 send_result = 0;
6577 }
6578 }
6579 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6580 call = frame->call;
6581 assert(call && &call->func->op_array == p->op_array);
6582
6583 if (opline->opcode == ZEND_DO_UCALL
6584 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6585 || opline->opcode == ZEND_DO_FCALL) {
6586
6587 frame->call_opline = opline;
6588
6589 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6590 if (opline > op_array->opcodes) {
6591 const zend_op *prev_opline = opline - 1;
6592
6593 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6594 prev_opline--;
6595 }
6596 JIT_G(current_frame) = call;
6597 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6598 || prev_opline->opcode == ZEND_SEND_UNPACK
6599 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6600 && p->op_array->num_args
6601 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6602 && ((p+1)->op == ZEND_JIT_TRACE_VM
6603 || (p+1)->op == ZEND_JIT_TRACE_END)
6604 && (TRACE_FRAME_NUM_ARGS(call) < 0
6605 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6606 && !zend_jit_trace_opline_guard(&dasm_state, (p+1)->opline)) {
6607 goto jit_failure;
6608 }
6609 JIT_G(current_frame) = frame;
6610 }
6611 }
6612
6613 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6614 p++;
6615 break;
6616 }
6617 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6618 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6619 TRACE_FRAME_SET_THIS_CHECKED(call);
6620 }
6621 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6622 TRACE_FRAME_SET_THIS_CHECKED(call);
6623 }
6624 op_array = (zend_op_array*)p->op_array;
6625 jit_extension =
6626 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6627 op_array_ssa = &jit_extension->func_info.ssa;
6628 frame->call = call->prev;
6629 call->prev = frame;
6630 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6631 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6632 } else {
6633 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6634 }
6635 JIT_G(current_frame) = frame = call;
6636 stack = frame->stack;
6637 if (ra) {
6638 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6639
6640 for (i = 0; i < op_array->last_var; i++,j++) {
6641 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6642 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6643 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6644 goto jit_failure;
6645 }
6646 }
6647 }
6648 }
6649 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6650 op_array = (zend_op_array*)p->op_array;
6651 jit_extension =
6652 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6653 op_array_ssa = &jit_extension->func_info.ssa;
6654 top = frame;
6655 if (frame->prev) {
6656 checked_stack = frame->old_checked_stack;
6657 peek_checked_stack = frame->old_peek_checked_stack;
6658 frame = frame->prev;
6659 stack = frame->stack;
6660 ZEND_ASSERT(&frame->func->op_array == op_array);
6661 } else {
6662 frame = zend_jit_trace_ret_frame(frame, op_array);
6663 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6664 frame->used_stack = checked_stack = peek_checked_stack = 0;
6665 stack = frame->stack;
6666 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6667 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6668
6669 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6670 /* Initialize abstract stack using SSA */
6671 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6672 && has_concrete_type(ssa->var_info[j].type)) {
6673 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6674 } else {
6675 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6676 }
6677 }
6678 if (ra) {
6679 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6680 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6681 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6682 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6683 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6684 goto jit_failure;
6685 }
6686 }
6687 }
6688 }
6689 } else {
6690 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6691 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6692 }
6693 }
6694 opline = NULL;
6695 }
6696 JIT_G(current_frame) = frame;
6697 if (res_type != IS_UNKNOWN
6698 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6699 const zend_op *opline = (p+1)->opline - 1;
6700 if (opline->result_type != IS_UNUSED) {
6701 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6702 }
6703 }
6704 res_type = IS_UNKNOWN;
6705 } else if (p->op == ZEND_JIT_TRACE_END) {
6706 break;
6707 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6708 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6709 int num_args = -1;
6710
6711 if (init_opline
6712 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6713 num_args = init_opline->extended_value;
6714 }
6715
6716 call = top;
6717 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6718 call->prev = frame->call;
6719 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6720 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6721 }
6722 if (init_opline
6723 && init_opline->opcode != ZEND_NEW
6724 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6725 || init_opline->op1_type == IS_UNDEF)
6726 && (init_opline->opcode != ZEND_INIT_USER_CALL
6727 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6728 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6729 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6730 ) {
6731 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6732 }
6733 frame->call = call;
6734 top = zend_jit_trace_call_frame(top, p->op_array);
6735 if (p->func) {
6736 if (p->func->type == ZEND_USER_FUNCTION) {
6737 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6738 zend_jit_op_array_trace_extension *jit_extension =
6739 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6740
6741 i = 0;
6742 while (i < p->op_array->num_args) {
6743 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6744 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6745 i++;
6746 }
6747 while (i < p->op_array->last_var) {
6748 if (jit_extension
6749 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6750 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6751 } else {
6752 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6753 }
6754 i++;
6755 }
6756 while (i < p->op_array->last_var + p->op_array->T) {
6757 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6758 i++;
6759 }
6760 } else {
6761 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
6762 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6763 }
6764 }
6765 } else {
6766 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
6767 for (i = 0; i < p->op_array->num_args; i++) {
6768 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6769 }
6770 }
6771 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6772 int skip_guard = 0;
6773
6774 if (init_opline) {
6775 zend_call_info *call_info = jit_extension->func_info.callee_info;
6776
6777 while (call_info) {
6778 if (call_info->caller_init_opline == init_opline
6779 && !call_info->is_prototype) {
6780 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
6781 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
6782 && init_opline->op1_type != IS_CONST) {
6783 break;
6784 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
6785 break;
6786 }
6787 }
6788 skip_guard = 1;
6789 break;
6790 }
6791 call_info = call_info->next_callee;
6792 }
6793 if (!skip_guard
6794 && !zend_jit_may_be_polymorphic_call(init_opline)
6795 && !zend_jit_may_be_modified(p->func, op_array)) {
6796 skip_guard = 1;
6797 }
6798 }
6799
6800 if (!skip_guard) {
6801 if (!opline) {
6802 zend_jit_trace_rec *q = p + 1;
6803 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
6804 q++;
6805 }
6806 opline = q->opline;
6807 ZEND_ASSERT(opline != NULL);
6808 }
6809 if (!zend_jit_init_fcall_guard(&dasm_state,
6810 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
6811 goto jit_failure;
6812 }
6813 }
6814 }
6815 }
6816 call->old_checked_stack = checked_stack;
6817 call->old_peek_checked_stack = peek_checked_stack;
6818 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6819 frame->call_level++;
6820 call->used_stack = checked_stack = peek_checked_stack = 0;
6821 } else {
6822 if (p->func) {
6823 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
6824 } else {
6825 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
6826 }
6827 switch (init_opline->opcode) {
6828 case ZEND_INIT_FCALL:
6829 case ZEND_INIT_FCALL_BY_NAME:
6830 case ZEND_INIT_NS_FCALL_BY_NAME:
6831 case ZEND_INIT_METHOD_CALL:
6832 case ZEND_INIT_DYNAMIC_CALL:
6833 //case ZEND_INIT_STATIC_METHOD_CALL:
6834 //case ZEND_INIT_USER_CALL:
6835 //case ZEND_NEW:
6836 checked_stack += call->used_stack;
6837 if (checked_stack > peek_checked_stack) {
6838 peek_checked_stack = checked_stack;
6839 }
6840 break;
6841 default:
6842 checked_stack = peek_checked_stack = 0;
6843 }
6844 }
6845 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
6846 call = frame->call;
6847 if (call) {
6848 checked_stack = call->old_checked_stack;
6849 peek_checked_stack = call->old_peek_checked_stack;
6850 top = call;
6851 frame->call = call->prev;
6852 }
6853 } else {
6854 ZEND_UNREACHABLE();
6855 }
6856 }
6857
6858 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
6859
6860 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
6861
6862 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6863 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6864 }
6865
6866 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
6867 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
6868 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6869 if (ra) {
6870 zend_ssa_phi *phi = ssa->blocks[1].phis;
6871
6872 while (phi) {
6873 if (ra[phi->ssa_var]
6874 && ra[phi->sources[1]]
6875 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
6876 && (ra[phi->ssa_var]->flags & (ZREG_LOAD|ZREG_STORE)) == 0
6877 && (ra[phi->sources[1]]->flags & (ZREG_LOAD|ZREG_STORE)) == 0) {
6878 /* Store actual type to memory to avoid deoptimization mistakes */
6879 /* TODO: Alternatively, we may try to update alredy generated deoptimization info */
6880 zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var));
6881 }
6882 phi = phi->next;
6883 }
6884 }
6885 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6886 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6887 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6888 goto jit_failure;
6889 }
6890 }
6891 t->link = ZEND_JIT_TRACE_NUM;
6892 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6893 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
6894 }
6895 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
6896 const void *timeout_exit_addr = NULL;
6897
6898 t->flags |= ZEND_JIT_TRACE_LOOP;
6899
6900 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6901 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6902 || (ra
6903 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
6904 /* Deoptimize to the first instruction of the loop */
6905 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
6906
6907 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6908 if (!timeout_exit_addr) {
6909 goto jit_failure;
6910 }
6911 } else {
6912 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6913 }
6914 }
6915
6916 zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
6917 }
6918 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
6919 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
6920 if (ra
6921 && (p-1)->op != ZEND_JIT_TRACE_ENTER
6922 && (p-1)->op != ZEND_JIT_TRACE_BACK
6923 && opline->opcode != ZEND_DO_UCALL
6924 && opline->opcode != ZEND_DO_FCALL
6925 && opline->opcode != ZEND_DO_FCALL_BY_NAME
6926 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
6927 if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL,
6928 stack, op_array->last_var + op_array->T, NULL, NULL, NULL, 0)) {
6929 goto jit_failure;
6930 }
6931 for (i = 0; i < op_array->last_var; i++) {
6932 int8_t reg = STACK_REG(stack, i);
6933 uint8_t type = STACK_TYPE(stack, i);
6934
6935 if (reg == ZREG_NONE
6936 && type != IS_UNKNOWN
6937 && type != STACK_MEM_TYPE(stack, i)) {
6938 if (!zend_jit_store_var_type(&dasm_state, i, type)) {
6939 return 0;
6940 }
6941 SET_STACK_TYPE(stack, i, type, 1);
6942 }
6943 }
6944 }
6945 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
6946 const void *timeout_exit_addr = NULL;
6947
6948 t->link = zend_jit_find_trace(p->opline->handler);
6949 if (t->link == 0) {
6950 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
6951 * by zend_jit_trace_exit() in another thread after this
6952 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
6953 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
6954 * the "_counter_handler" functions, and these are not registered
6955 * tracer functions */
6956 goto jit_failure;
6957 }
6958 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6959 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6960 goto jit_failure;
6961 }
6962 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6963 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6964 }
6965 if (parent_trace
6966 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
6967 && zend_jit_traces[parent_trace].root == t->link) {
6968 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
6969 uint32_t exit_point;
6970
6971 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6972 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6973 }
6974 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
6975 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6976 if (!timeout_exit_addr) {
6977 goto jit_failure;
6978 }
6979 } else {
6980 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6981 }
6982 }
6983 zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
6984 } else {
6985 zend_jit_trace_return(&dasm_state, 0, NULL);
6986 }
6987 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
6988 zend_jit_trace_return(&dasm_state, 0, NULL);
6989 } else {
6990 // TODO: not implemented ???
6991 ZEND_ASSERT(0 && p->stop);
6992 }
6993
6994 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
6995 goto jit_failure;
6996 }
6997
6998 if (!zend_jit_trace_end(&dasm_state, t)) {
6999 goto jit_failure;
7000 }
7001
7002 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM,
7003 parent_trace ? SP_ADJ_JIT : ((zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET),
7004 parent_trace ? SP_ADJ_NONE : SP_ADJ_JIT);
7005
7006 if (handler) {
7007 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
7008 const zend_op_array *rec_op_array;
7009
7010 rec_op_array = op_array = trace_buffer->op_array;
7011 jit_extension =
7012 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7013 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7014 for (;;p++) {
7015 if (p->op == ZEND_JIT_TRACE_VM) {
7016 opline = p->opline;
7017 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7018 if (p->op_array == rec_op_array) {
7019 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7020 }
7021 op_array = p->op_array;
7022 jit_extension =
7023 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7024 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7025 op_array = p->op_array;
7026 jit_extension =
7027 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7028 } else if (p->op == ZEND_JIT_TRACE_END) {
7029 break;
7030 }
7031 }
7032 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7033 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7034 if (opline
7035 && (opline->opcode == ZEND_DO_UCALL
7036 || opline->opcode == ZEND_DO_FCALL
7037 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7038 || opline->opcode == ZEND_YIELD
7039 || opline->opcode == ZEND_YIELD_FROM
7040 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7041 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7042 }
7043 if (JIT_G(current_frame)
7044 && JIT_G(current_frame)->prev) {
7045 frame = JIT_G(current_frame)->prev;
7046 do {
7047 if (frame->call_opline) {
7048 op_array = &frame->func->op_array;
7049 jit_extension =
7050 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7051 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7052 }
7053 frame = frame->prev;
7054 } while (frame);
7055 }
7056 }
7057 }
7058
7059 jit_failure:
7060 dasm_free(&dasm_state);
7061
7062 if (name) {
7063 zend_string_release(name);
7064 }
7065
7066 jit_cleanup:
7067 /* Clean up used op_arrays */
7068 while (num_op_arrays > 0) {
7069 op_array = op_arrays[--num_op_arrays];
7070 jit_extension =
7071 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7072
7073 jit_extension->func_info.num = 0;
7074 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7075 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7076 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7077 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7078 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7079 }
7080
7081 zend_arena_release(&CG(arena), checkpoint);
7082
7083 JIT_G(current_frame) = NULL;
7084 JIT_G(current_trace) = NULL;
7085
7086 return handler;
7087 }
7088
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7089 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7090 {
7091 const void *handler = NULL;
7092 dasm_State* dasm_state = NULL;
7093 void *checkpoint;
7094 char name[32];
7095 const zend_op *opline;
7096 uint32_t stack_size;
7097 zend_jit_trace_stack *stack;
7098 bool original_handler = 0;
7099
7100 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7101 return dasm_labels[zend_lbtrace_escape];
7102 }
7103
7104 checkpoint = zend_arena_checkpoint(CG(arena));;
7105
7106 sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num);
7107
7108 dasm_init(&dasm_state, DASM_MAXSECTION);
7109 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
7110 dasm_setup(&dasm_state, dasm_actions);
7111
7112 zend_jit_align_func(&dasm_state);
7113
7114 /* Deoptimization */
7115 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7116 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
7117
7118 if (!zend_jit_trace_deoptimization(&dasm_state,
7119 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7120 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7121 stack, stack_size, NULL, NULL, NULL, 0)) {
7122 goto jit_failure;
7123 }
7124
7125 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7126 if (opline) {
7127 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7128 /* prevent endless loop */
7129 original_handler = 1;
7130 }
7131 zend_jit_set_ip_ex(&dasm_state, opline, original_handler);
7132 }
7133
7134 zend_jit_trace_return(&dasm_state, original_handler, opline);
7135
7136 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM, SP_ADJ_JIT, SP_ADJ_NONE);
7137
7138 jit_failure:
7139 dasm_free(&dasm_state);
7140 zend_arena_release(&CG(arena), checkpoint);
7141 return handler;
7142 }
7143
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7144 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7145 {
7146 zend_jit_trace_stop ret;
7147 const void *handler;
7148 uint8_t orig_trigger;
7149 zend_jit_trace_info *t = NULL;
7150 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7151 bool do_bailout = 0;
7152
7153 zend_shared_alloc_lock();
7154
7155 /* Checks under lock */
7156 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7157 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7158 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7159 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7160 } else {
7161 zend_try {
7162 SHM_UNPROTECT();
7163 zend_jit_unprotect();
7164
7165 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7166
7167 t->id = ZEND_JIT_TRACE_NUM;
7168 t->root = ZEND_JIT_TRACE_NUM;
7169 t->parent = 0;
7170 t->link = 0;
7171 t->exit_count = 0;
7172 t->child_count = 0;
7173 t->stack_map_size = 0;
7174 t->flags = 0;
7175 t->polymorphism = 0;
7176 t->jmp_table_size = 0;
7177 t->op_array = trace_buffer[0].op_array;
7178 t->opline = trace_buffer[1].opline;
7179 t->exit_info = exit_info;
7180 t->stack_map = NULL;
7181
7182 orig_trigger = JIT_G(trigger);
7183 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7184
7185 handler = zend_jit_trace(trace_buffer, 0, 0);
7186
7187 JIT_G(trigger) = orig_trigger;
7188
7189 if (handler) {
7190 zend_jit_trace_exit_info *shared_exit_info = NULL;
7191
7192 t->exit_info = NULL;
7193 if (t->exit_count) {
7194 /* reallocate exit_info into shared memory */
7195 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7196 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7197
7198 if (!shared_exit_info) {
7199 if (t->stack_map) {
7200 efree(t->stack_map);
7201 t->stack_map = NULL;
7202 }
7203 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7204 goto exit;
7205 }
7206 memcpy(shared_exit_info, exit_info,
7207 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7208 t->exit_info = shared_exit_info;
7209 }
7210
7211 if (t->stack_map_size) {
7212 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7213 if (!shared_stack_map) {
7214 efree(t->stack_map);
7215 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7216 goto exit;
7217 }
7218 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7219 efree(t->stack_map);
7220 t->stack_map = shared_stack_map;
7221 }
7222
7223 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7224 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7225
7226 ((zend_op*)opline)->handler = handler;
7227
7228 ZEND_JIT_TRACE_NUM++;
7229 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7230
7231 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7232 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7233 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7234 if (t->stack_map) {
7235 efree(t->stack_map);
7236 t->stack_map = NULL;
7237 }
7238 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7239 } else {
7240 if (t->stack_map) {
7241 efree(t->stack_map);
7242 t->stack_map = NULL;
7243 }
7244 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7245 }
7246
7247 exit:;
7248 } zend_catch {
7249 do_bailout = 1;
7250 } zend_end_try();
7251
7252 zend_jit_protect();
7253 SHM_PROTECT();
7254 }
7255
7256 zend_shared_alloc_unlock();
7257
7258 if (do_bailout) {
7259 zend_bailout();
7260 }
7261
7262 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7263 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7264 && t->exit_count > 0) {
7265 zend_jit_dump_exit_info(t);
7266 }
7267
7268 return ret;
7269 }
7270
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7271 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7272 {
7273 zend_shared_alloc_lock();
7274
7275 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7276 SHM_UNPROTECT();
7277 zend_jit_unprotect();
7278
7279 ((zend_op*)opline)->handler =
7280 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7281
7282 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7283
7284 zend_jit_protect();
7285 SHM_PROTECT();
7286 }
7287
7288 zend_shared_alloc_unlock();
7289 }
7290
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7291 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7292 {
7293 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7294 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7295 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7296 uint32_t cache_slot = JIT_G(bad_root_slot);
7297 uint32_t i;
7298
7299 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7300 if (cache_opline[i] == opline) {
7301 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7302 cache_opline[i] = NULL;
7303 return 1;
7304 } else {
7305 #if 0
7306 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7307 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7308 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7309 }
7310 #endif
7311 cache_count[i]++;
7312 cache_stop[i] = stop;
7313 return 0;
7314 }
7315 }
7316 }
7317 i = cache_slot;
7318 cache_opline[i] = opline;
7319 cache_count[i] = 1;
7320 cache_stop[i] = stop;
7321 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7322 JIT_G(bad_root_slot) = cache_slot;
7323 return 0;
7324 }
7325
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7326 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7327 {
7328 zend_jit_trace_rec *p = trace_buffer;
7329 const zend_op_array *op_array;
7330 const zend_op *opline;
7331 uint32_t level = 1 + trace_buffer[0].level;
7332 int idx, len, i, v, vars_count, call_level;
7333
7334 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7335 op_array = p->op_array;
7336 p += ZEND_JIT_TRACE_START_REC_SIZE;
7337 idx = 0;
7338 call_level = 0;
7339
7340 if (tssa && tssa->var_info) {
7341 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7342 vars_count = op_array->last_var;
7343 } else {
7344 vars_count = op_array->last_var + op_array->T;
7345 }
7346 for (i = 0; i < vars_count; i++) {
7347 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7348 fprintf(stderr, " %*c;", level, ' ');
7349 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7350 fprintf(stderr, "\n");
7351 }
7352 }
7353 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7354 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7355 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7356 zend_ssa_phi *p = tssa->blocks[1].phis;
7357
7358 fprintf(stderr, "LOOP:\n");
7359
7360 while (p) {
7361 fprintf(stderr, " ;");
7362 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7363 fprintf(stderr, " = Phi(");
7364 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7365 fprintf(stderr, ", ");
7366 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7367 fprintf(stderr, ")\n");
7368 p = p->next;
7369 }
7370 }
7371 }
7372
7373 while (1) {
7374 if (p->op == ZEND_JIT_TRACE_VM) {
7375 uint8_t op1_type, op2_type, op3_type;
7376
7377 opline = p->opline;
7378 fprintf(stderr, "%04d%*c",
7379 (int)(opline - op_array->opcodes),
7380 level, ' ');
7381 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7382
7383 op1_type = p->op1_type;
7384 op2_type = p->op2_type;
7385 op3_type = p->op3_type;
7386 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7387 fprintf(stderr, " ;");
7388 if (op1_type != IS_UNKNOWN) {
7389 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7390 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7391 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7392 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7393 p++;
7394 fprintf(stderr, " op1(%sobject of class %s)", ref,
7395 ZSTR_VAL(p->ce->name));
7396 } else {
7397 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7398 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7399 }
7400 }
7401 if (op2_type != IS_UNKNOWN) {
7402 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7403 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7404 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7405 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7406 p++;
7407 fprintf(stderr, " op2(%sobject of class %s)", ref,
7408 ZSTR_VAL(p->ce->name));
7409 } else {
7410 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7411 fprintf(stderr, " op2(%s%s)", ref, type);
7412 }
7413 }
7414 if (op3_type != IS_UNKNOWN) {
7415 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7416 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7417 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7418 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7419 fprintf(stderr, " op3(%s%s)", ref, type);
7420 }
7421 }
7422 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7423 uint8_t val_type;
7424 const char *type;
7425
7426 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7427 fprintf(stderr, " ;");
7428 }
7429 p++;
7430 val_type = p->op1_type;
7431
7432 if (val_type == IS_UNDEF) {
7433 type = "undef";
7434 } else if (val_type == IS_REFERENCE) {
7435 type = "ref";
7436 } else {
7437 type = zend_get_type_by_const(val_type);
7438 }
7439 fprintf(stderr, " val(%s)", type);
7440 }
7441 fprintf(stderr, "\n");
7442 idx++;
7443
7444 len = zend_jit_trace_op_len(opline);
7445 while (len > 1) {
7446 opline++;
7447 fprintf(stderr, "%04d%*c;",
7448 (int)(opline - op_array->opcodes),
7449 level, ' ');
7450 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7451 idx++;
7452 len--;
7453 fprintf(stderr, "\n");
7454 }
7455 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7456 op_array = p->op_array;
7457 fprintf(stderr, " %*c>enter %s%s%s\n",
7458 level, ' ',
7459 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7460 op_array->scope ? "::" : "",
7461 op_array->function_name ?
7462 ZSTR_VAL(op_array->function_name) :
7463 ZSTR_VAL(op_array->filename));
7464 level++;
7465 if (tssa && tssa->var_info) {
7466 call_level++;
7467 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7468 vars_count = op_array->last_var;
7469 for (i = 0; i < vars_count; i++, v++) {
7470 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7471 fprintf(stderr, " %*c;", level, ' ');
7472 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7473 fprintf(stderr, "\n");
7474 }
7475 }
7476 }
7477 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7478 op_array = p->op_array;
7479 level--;
7480 fprintf(stderr, " %*c<back %s%s%s\n",
7481 level, ' ',
7482 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7483 op_array->scope ? "::" : "",
7484 op_array->function_name ?
7485 ZSTR_VAL(op_array->function_name) :
7486 ZSTR_VAL(op_array->filename));
7487 if (tssa && tssa->var_info) {
7488 if (call_level == 0) {
7489 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7490 vars_count = op_array->last_var + op_array->T;
7491 for (i = 0; i < vars_count; i++, v++) {
7492 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7493 fprintf(stderr, " %*c;", level, ' ');
7494 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7495 fprintf(stderr, "\n");
7496 }
7497 }
7498 } else {
7499 call_level--;
7500 }
7501 }
7502 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7503 if (p->func != (zend_function*)&zend_pass_function) {
7504 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7505 level, ' ',
7506 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7507 (p->func && p->func->common.scope) ? "::" : "",
7508 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7509 } else {
7510 fprintf(stderr, " %*c>skip\n",
7511 level, ' ');
7512 }
7513 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7514 if (p->func != (zend_function*)&zend_pass_function) {
7515 fprintf(stderr, " %*c>call %s%s%s\n",
7516 level, ' ',
7517 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7518 p->func->common.scope ? "::" : "",
7519 ZSTR_VAL(p->func->common.function_name));
7520 } else {
7521 fprintf(stderr, " %*c>skip\n",
7522 level, ' ');
7523 }
7524 } else if (p->op == ZEND_JIT_TRACE_END) {
7525 break;
7526 }
7527 p++;
7528 }
7529 }
7530
zend_jit_dump_exit_info(zend_jit_trace_info * t)7531 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7532 {
7533 int i, j;
7534
7535 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7536 for (i = 0; i < t->exit_count; i++) {
7537 const zend_op_array *op_array = t->exit_info[i].op_array;
7538 uint32_t stack_size = t->exit_info[i].stack_size;
7539 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset;
7540
7541 fprintf(stderr, " exit_%d:", i);
7542 if (t->exit_info[i].opline) {
7543 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7544 } else {
7545 fprintf(stderr, " ----/");
7546 }
7547 if (t->exit_info[i].stack_size) {
7548 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7549 } else {
7550 fprintf(stderr, "----/0");
7551 }
7552 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7553 fprintf(stderr, "/VM");
7554 }
7555 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7556 fprintf(stderr, "/CALL");
7557 }
7558 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7559 fprintf(stderr, "/POLY");
7560 }
7561 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7562 fprintf(stderr, "/FREE_OP1");
7563 }
7564 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7565 fprintf(stderr, "/FREE_OP2");
7566 }
7567 for (j = 0; j < stack_size; j++) {
7568 zend_uchar type = STACK_TYPE(stack, j);
7569 if (type != IS_UNKNOWN) {
7570 fprintf(stderr, " ");
7571 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7572 fprintf(stderr, ":");
7573 if (type == IS_UNDEF) {
7574 fprintf(stderr, "undef");
7575 } else {
7576 fprintf(stderr, "%s", zend_get_type_by_const(type));
7577 }
7578 if (STACK_REG(stack, j) != ZREG_NONE) {
7579 if (STACK_REG(stack, j) < ZREG_NUM) {
7580 fprintf(stderr, "(%s)", zend_reg_name[STACK_REG(stack, j)]);
7581 } else if (STACK_REG(stack, j) == ZREG_THIS) {
7582 fprintf(stderr, "(this)");
7583 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7584 fprintf(stderr, "(zval_try_addref)");
7585 } else {
7586 fprintf(stderr, "(const_%d)", STACK_REG(stack, j) - ZREG_NUM);
7587 }
7588 }
7589 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7590 fprintf(stderr, " ");
7591 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7592 fprintf(stderr, ":unknown(zval_try_addref)");
7593 } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_GPR0) {
7594 fprintf(stderr, " ");
7595 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7596 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[ZREG_COPY]);
7597 }
7598 }
7599 fprintf(stderr, "\n");
7600 }
7601 }
7602
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)7603 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
7604 {
7605 const zend_op *orig_opline;
7606 zend_jit_trace_stop stop;
7607 int ret = 0;
7608 zend_op_array *op_array;
7609 zend_jit_op_array_trace_extension *jit_extension;
7610 size_t offset;
7611 uint32_t trace_num;
7612 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7613
7614 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
7615 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
7616 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
7617
7618 repeat:
7619 trace_num = ZEND_JIT_TRACE_NUM;
7620 orig_opline = opline;
7621 op_array = &EX(func)->op_array;
7622 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7623 offset = jit_extension->offset;
7624
7625 EX(opline) = opline;
7626
7627 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
7628 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7629 return 0;
7630 }
7631
7632 if (JIT_G(tracing)) {
7633 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
7634 return 0;
7635 }
7636
7637 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7638 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7639 trace_num,
7640 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7641 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7642 EX(func)->op_array.scope ? "::" : "",
7643 EX(func)->op_array.function_name ?
7644 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7645 ZSTR_VAL(EX(func)->op_array.filename),
7646 opline->lineno);
7647 }
7648
7649 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7650 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7651 goto abort;
7652 }
7653
7654 JIT_G(tracing) = 1;
7655 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
7656 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
7657 JIT_G(tracing) = 0;
7658
7659 if (stop & ZEND_JIT_TRACE_HALT) {
7660 ret = -1;
7661 }
7662 stop &= ~ZEND_JIT_TRACE_HALT;
7663
7664 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
7665 orig_opline = trace_buffer[1].opline;
7666 op_array = (zend_op_array*)trace_buffer[0].op_array;
7667 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7668 offset = jit_extension->offset;
7669 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7670 const zend_op_array *op_array = trace_buffer[0].op_array;
7671 const zend_op *opline = trace_buffer[1].opline;
7672 zend_jit_op_array_trace_extension *jit_extension =
7673 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7674 size_t offset = jit_extension->offset;
7675
7676 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7677 trace_num,
7678 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7679 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7680 op_array->scope ? "::" : "",
7681 op_array->function_name ?
7682 ZSTR_VAL(op_array->function_name) : "$main",
7683 ZSTR_VAL(op_array->filename),
7684 opline->lineno);
7685 }
7686 }
7687
7688 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
7689 zend_jit_dump_trace(trace_buffer, NULL);
7690 }
7691
7692 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
7693 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
7694 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
7695 uint32_t idx = trace_buffer[1].last;
7696 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
7697 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
7698 trace_num,
7699 link_to);
7700 } else {
7701 fprintf(stderr, "---- TRACE %d stop (%s)\n",
7702 trace_num,
7703 zend_jit_trace_stop_description[stop]);
7704 }
7705 }
7706 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
7707 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
7708 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
7709 fprintf(stderr, "---- TRACE %d %s\n",
7710 trace_num,
7711 zend_jit_trace_stop_description[stop]);
7712 }
7713 } else {
7714 goto abort;
7715 }
7716 } else {
7717 abort:
7718 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
7719 fprintf(stderr, "---- TRACE %d abort (%s)\n",
7720 trace_num,
7721 zend_jit_trace_stop_description[stop]);
7722 }
7723 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
7724 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
7725 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7726 fprintf(stderr, "---- TRACE %d blacklisted\n",
7727 trace_num);
7728 }
7729 zend_jit_blacklist_root_trace(orig_opline, offset);
7730 }
7731 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
7732 execute_data = EG(current_execute_data);
7733 opline = EX(opline);
7734 goto repeat;
7735 }
7736 }
7737
7738 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
7739 fprintf(stderr, "\n");
7740 }
7741
7742 return ret;
7743 }
7744
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)7745 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
7746 {
7747 const void *handler;
7748 bool do_bailout = 0;
7749
7750 zend_shared_alloc_lock();
7751
7752 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
7753 SHM_UNPROTECT();
7754 zend_jit_unprotect();
7755
7756 zend_try {
7757 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
7758
7759 if (handler) {
7760 zend_jit_link_side_trace(
7761 zend_jit_traces[trace_num].code_start,
7762 zend_jit_traces[trace_num].code_size,
7763 zend_jit_traces[trace_num].jmp_table_size,
7764 exit_num,
7765 handler);
7766 }
7767 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
7768 } zend_catch {
7769 do_bailout = 1;
7770 } zend_end_try();
7771
7772 zend_jit_protect();
7773 SHM_PROTECT();
7774 }
7775
7776 zend_shared_alloc_unlock();
7777
7778 if (do_bailout) {
7779 zend_bailout();
7780 }
7781 }
7782
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)7783 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
7784 {
7785 uint8_t *counter = JIT_G(exit_counters) +
7786 zend_jit_traces[trace_num].exit_counters + exit_num;
7787
7788 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
7789 return 1;
7790 }
7791 (*counter)++;
7792 return 0;
7793 }
7794
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)7795 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
7796 {
7797 uint8_t *counter = JIT_G(exit_counters) +
7798 zend_jit_traces[trace_num].exit_counters + exit_num;
7799
7800 if (*counter + 1 >= JIT_G(hot_side_exit)) {
7801 return 1;
7802 }
7803 (*counter)++;
7804 return 0;
7805 }
7806
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)7807 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
7808 {
7809 zend_jit_trace_stop ret;
7810 const void *handler;
7811 uint8_t orig_trigger;
7812 zend_jit_trace_info *t;
7813 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7814 bool do_bailout = 0;
7815
7816 zend_shared_alloc_lock();
7817
7818 /* Checks under lock */
7819 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7820 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7821 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7822 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7823 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
7824 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
7825 } else {
7826 SHM_UNPROTECT();
7827 zend_jit_unprotect();
7828
7829 zend_try {
7830 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7831
7832 t->id = ZEND_JIT_TRACE_NUM;
7833 t->root = zend_jit_traces[parent_num].root;
7834 t->parent = parent_num;
7835 t->link = 0;
7836 t->exit_count = 0;
7837 t->child_count = 0;
7838 t->stack_map_size = 0;
7839 t->flags = 0;
7840 t->polymorphism = polymorphism;
7841 t->jmp_table_size = 0;
7842 t->opline = NULL;
7843 t->exit_info = exit_info;
7844 t->stack_map = NULL;
7845
7846 orig_trigger = JIT_G(trigger);
7847 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7848
7849 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
7850
7851 JIT_G(trigger) = orig_trigger;
7852
7853 if (handler) {
7854 zend_jit_trace_exit_info *shared_exit_info = NULL;
7855
7856 t->exit_info = NULL;
7857 if (t->exit_count) {
7858 /* reallocate exit_info into shared memory */
7859 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7860 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7861
7862 if (!shared_exit_info) {
7863 if (t->stack_map) {
7864 efree(t->stack_map);
7865 t->stack_map = NULL;
7866 }
7867 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7868 goto exit;
7869 }
7870 memcpy(shared_exit_info, exit_info,
7871 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7872 t->exit_info = shared_exit_info;
7873 }
7874
7875 if (t->stack_map_size) {
7876 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7877 if (!shared_stack_map) {
7878 efree(t->stack_map);
7879 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7880 goto exit;
7881 }
7882 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7883 efree(t->stack_map);
7884 t->stack_map = shared_stack_map;
7885 }
7886
7887 zend_jit_link_side_trace(
7888 zend_jit_traces[parent_num].code_start,
7889 zend_jit_traces[parent_num].code_size,
7890 zend_jit_traces[parent_num].jmp_table_size,
7891 exit_num,
7892 handler);
7893
7894 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7895 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7896
7897 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
7898 ZEND_JIT_TRACE_NUM++;
7899 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
7900
7901 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7902 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7903 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7904 if (t->stack_map) {
7905 efree(t->stack_map);
7906 t->stack_map = NULL;
7907 }
7908 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7909 } else {
7910 if (t->stack_map) {
7911 efree(t->stack_map);
7912 t->stack_map = NULL;
7913 }
7914 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7915 }
7916
7917 exit:;
7918 } zend_catch {
7919 do_bailout = 1;
7920 } zend_end_try();
7921
7922 zend_jit_protect();
7923 SHM_PROTECT();
7924 }
7925
7926 zend_shared_alloc_unlock();
7927
7928 if (do_bailout) {
7929 zend_bailout();
7930 }
7931
7932 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7933 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7934 && t->exit_count > 0) {
7935 zend_jit_dump_exit_info(t);
7936 }
7937
7938 return ret;
7939 }
7940
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)7941 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
7942 {
7943 zend_jit_trace_stop stop;
7944 int ret = 0;
7945 uint32_t trace_num;
7946 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7947 uint32_t is_megamorphic = 0;
7948 uint32_t polymorphism = 0;
7949
7950 trace_num = ZEND_JIT_TRACE_NUM;
7951
7952 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
7953 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7954 return 0;
7955 }
7956
7957 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7958 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
7959 trace_num, parent_num, exit_num,
7960 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7961 EX(func)->op_array.scope ? "::" : "",
7962 EX(func)->op_array.function_name ?
7963 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7964 ZSTR_VAL(EX(func)->op_array.filename),
7965 EX(opline)->lineno);
7966 }
7967
7968 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7969 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7970 goto abort;
7971 }
7972
7973 if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
7974 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
7975 goto abort;
7976 }
7977
7978 if (JIT_G(max_polymorphic_calls) > 0) {
7979 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
7980 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
7981 && EX(call))) {
7982 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
7983 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
7984 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
7985 } else if (!zend_jit_traces[parent_num].polymorphism) {
7986 polymorphism = 1;
7987 } else if (exit_num == 0) {
7988 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
7989 }
7990 }
7991 }
7992
7993 JIT_G(tracing) = 1;
7994 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
7995 JIT_G(tracing) = 0;
7996
7997 if (stop & ZEND_JIT_TRACE_HALT) {
7998 ret = -1;
7999 }
8000 stop &= ~ZEND_JIT_TRACE_HALT;
8001
8002 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8003 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8004 const zend_op_array *op_array = trace_buffer[0].op_array;
8005 const zend_op *opline = trace_buffer[1].opline;
8006 zend_jit_op_array_trace_extension *jit_extension =
8007 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8008 size_t offset = jit_extension->offset;
8009
8010 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8011 trace_num,
8012 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8013 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8014 op_array->scope ? "::" : "",
8015 op_array->function_name ?
8016 ZSTR_VAL(op_array->function_name) : "$main",
8017 ZSTR_VAL(op_array->filename),
8018 opline->lineno);
8019 }
8020 }
8021
8022 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8023 zend_jit_dump_trace(trace_buffer, NULL);
8024 }
8025
8026 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8027 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8028 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8029 uint32_t idx = trace_buffer[1].last;
8030 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8031 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8032 trace_num,
8033 link_to);
8034 } else {
8035 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8036 trace_num,
8037 zend_jit_trace_stop_description[stop]);
8038 }
8039 }
8040 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8041 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8042 } else {
8043 const zend_op_array *op_array = trace_buffer[0].op_array;
8044 zend_jit_op_array_trace_extension *jit_extension =
8045 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8046 const zend_op *opline = trace_buffer[1].opline;
8047
8048 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8049 }
8050 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8051 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8052 fprintf(stderr, "---- TRACE %d %s\n",
8053 trace_num,
8054 zend_jit_trace_stop_description[stop]);
8055 }
8056 } else {
8057 goto abort;
8058 }
8059 } else {
8060 abort:
8061 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8062 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8063 trace_num,
8064 zend_jit_trace_stop_description[stop]);
8065 }
8066 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8067 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8068 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8069 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8070 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8071 parent_num, exit_num);
8072 }
8073 }
8074 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8075 execute_data = EG(current_execute_data);
8076 return zend_jit_trace_hot_root(execute_data, EX(opline));
8077 }
8078 }
8079
8080 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8081 fprintf(stderr, "\n");
8082 }
8083
8084 return ret;
8085 }
8086
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8087 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8088 {
8089 uint32_t trace_num = EG(jit_trace_num);
8090 zend_execute_data *execute_data = EG(current_execute_data);
8091 const zend_op *orig_opline = EX(opline);
8092 const zend_op *opline;
8093 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8094 int repeat_last_opline = 0;
8095
8096 /* Deoptimizatoion of VM stack state */
8097 uint32_t i;
8098 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8099 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset;
8100
8101 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8102 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8103 call->prev_execute_data = EX(call);
8104 EX(call) = call;
8105 }
8106
8107 for (i = 0; i < stack_size; i++) {
8108 if (STACK_REG(stack, i) != ZREG_NONE) {
8109 if (STACK_TYPE(stack, i) == IS_LONG) {
8110 zend_long val;
8111
8112 if (STACK_REG(stack, i) < ZREG_NUM) {
8113 val = regs->gpr[STACK_REG(stack, i)];
8114 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN) {
8115 val = ZEND_LONG_MIN;
8116 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX) {
8117 val = ZEND_LONG_MAX;
8118 } else {
8119 ZEND_UNREACHABLE();
8120 }
8121 ZVAL_LONG(EX_VAR_NUM(i), val);
8122 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8123 double val;
8124
8125 if (STACK_REG(stack, i) < ZREG_NUM) {
8126 val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8127 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN_MINUS_1) {
8128 val = (double)ZEND_LONG_MIN - 1.0;
8129 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX_PLUS_1) {
8130 val = (double)ZEND_LONG_MAX + 1.0;
8131 } else {
8132 ZEND_UNREACHABLE();
8133 }
8134 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8135 } else if (STACK_REG(stack, i) == ZREG_THIS) {
8136 zend_object *obj = Z_OBJ(EX(This));
8137
8138 GC_ADDREF(obj);
8139 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8140 } else if (STACK_REG(stack, i) == ZREG_NULL) {
8141 ZVAL_NULL(EX_VAR_NUM(i));
8142 } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) {
8143 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8144 } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_GPR0) {
8145 zval *val = (zval*)regs->gpr[ZREG_COPY];
8146
8147 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8148 /* Undefined array index or property */
8149 repeat_last_opline = 1;
8150 } else {
8151 ZVAL_COPY(EX_VAR_NUM(i), val);
8152 }
8153 } else {
8154 ZEND_UNREACHABLE();
8155 }
8156 }
8157 }
8158
8159 if (repeat_last_opline) {
8160 EX(opline) = t->exit_info[exit_num].opline - 1;
8161 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8162 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8163 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8164 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8165 }
8166 return 1;
8167 }
8168
8169 opline = t->exit_info[exit_num].opline;
8170
8171 if (opline) {
8172 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8173 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8174 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8175 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8176 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8177 EX(opline) = opline-1;
8178 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8179 }
8180 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8181 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8182 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8183 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8184 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8185 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8186 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8187 EX(opline) = opline-1;
8188 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8189 }
8190 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8191 if (EG(exception)) {
8192 return 1;
8193 }
8194 }
8195 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8196 zend_function *func = (zend_function*)regs->gpr[ZREG_COPY];
8197
8198 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8199 zend_string_release_ex(func->common.function_name, 0);
8200 zend_free_trampoline(func);
8201 EX(opline) = opline;
8202 return 1;
8203 }
8204 }
8205
8206 /* Set VM opline to continue interpretation */
8207 EX(opline) = opline;
8208 }
8209
8210 if (EG(vm_interrupt) || JIT_G(tracing)) {
8211 return 1;
8212 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8213 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8214 return 0;
8215 }
8216
8217 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8218 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8219 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8220
8221 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8222 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8223 trace_num,
8224 exit_num,
8225 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8226 EX(func)->op_array.scope ? "::" : "",
8227 EX(func)->op_array.function_name ?
8228 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8229 ZSTR_VAL(EX(func)->op_array.filename),
8230 EX(opline)->lineno);
8231 }
8232
8233 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8234 zend_jit_op_array_trace_extension *jit_extension;
8235 uint32_t num = trace_num;
8236
8237 while (t->root != num) {
8238 num = t->root;
8239 t = &zend_jit_traces[num];
8240 }
8241
8242 zend_shared_alloc_lock();
8243
8244 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8245
8246 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8247 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8248 /* skip: not JIT-ed nor blacklisted */
8249 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8250 /* skip: too many root traces */
8251 } else {
8252 SHM_UNPROTECT();
8253 zend_jit_unprotect();
8254
8255 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8256 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8257 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8258 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8259 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8260 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8261 }
8262 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8263 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8264
8265 zend_jit_protect();
8266 SHM_PROTECT();
8267 }
8268
8269 zend_shared_alloc_unlock();
8270
8271 return 0;
8272 }
8273
8274 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8275 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8276 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8277 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8278 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8279 trace_num, exit_num);
8280 }
8281 return 0;
8282 }
8283 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8284 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8285 }
8286
8287 /* Return 1 to call original handler instead of the same JIT-ed trace */
8288 return (orig_opline == t->opline && EX(opline) == orig_opline);
8289 }
8290
zend_jit_trace_supported(const zend_op * opline)8291 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8292 {
8293 switch (opline->opcode) {
8294 case ZEND_CATCH:
8295 case ZEND_FAST_CALL:
8296 case ZEND_FAST_RET:
8297 return ZEND_JIT_TRACE_UNSUPPORTED;
8298 default:
8299 return ZEND_JIT_TRACE_SUPPORTED;
8300 }
8301 }
8302
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8303 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8304 {
8305 zend_jit_op_array_trace_extension *jit_extension;
8306 uint32_t i;
8307
8308 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8309 for (i = 0; i < op_array->last; i++) {
8310 jit_extension->trace_info[i].trace_flags &=
8311 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8312 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8313 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8314 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8315 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8316 } else {
8317 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8318 }
8319 }
8320 return SUCCESS;
8321 }
8322
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8323 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8324 {
8325 zend_op *opline;
8326 zend_jit_op_array_trace_extension *jit_extension;
8327 uint32_t i;
8328
8329 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8330
8331 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8332 if (!jit_extension) {
8333 return FAILURE;
8334 }
8335 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8336 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8337 jit_extension->op_array = op_array;
8338 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8339 for (i = 0; i < op_array->last; i++) {
8340 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8341 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8342 jit_extension->trace_info[i].counter = NULL;
8343 jit_extension->trace_info[i].trace_flags =
8344 zend_jit_trace_supported(&op_array->opcodes[i]);
8345 }
8346 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8347
8348 if (JIT_G(hot_loop)) {
8349 zend_cfg cfg;
8350
8351 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8352
8353 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8354 return FAILURE;
8355 }
8356
8357 for (i = 0; i < cfg.blocks_count; i++) {
8358 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8359 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8360 /* loop header */
8361 opline = op_array->opcodes + cfg.blocks[i].start;
8362 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8363 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8364 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8365 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8366 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8367 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8368 }
8369 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8370 ZEND_JIT_TRACE_START_LOOP;
8371 }
8372 }
8373 }
8374 }
8375 }
8376
8377 if (JIT_G(hot_func)) {
8378 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8379 opline = op_array->opcodes;
8380 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8381 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8382 opline++;
8383 }
8384 }
8385
8386 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8387 /* function entry */
8388 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8389 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8390 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8391 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8392 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8393 ZEND_JIT_TRACE_START_ENTER;
8394 }
8395 }
8396
8397 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8398
8399 return SUCCESS;
8400 }
8401
zend_jit_trace_init_caches(void)8402 static void zend_jit_trace_init_caches(void)
8403 {
8404 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8405 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8406 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8407 JIT_G(bad_root_slot) = 0;
8408
8409 if (JIT_G(exit_counters)) {
8410 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8411 }
8412 }
8413
zend_jit_trace_reset_caches(void)8414 static void zend_jit_trace_reset_caches(void)
8415 {
8416 JIT_G(tracing) = 0;
8417 #ifdef ZTS
8418 if (!JIT_G(exit_counters)) {
8419 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8420 }
8421 #endif
8422 }
8423
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8424 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8425 {
8426 if (jit_globals->exit_counters) {
8427 free(jit_globals->exit_counters);
8428 }
8429 }
8430
zend_jit_trace_restart(void)8431 static void zend_jit_trace_restart(void)
8432 {
8433 ZEND_JIT_TRACE_NUM = 1;
8434 ZEND_JIT_COUNTER_NUM = 0;
8435 ZEND_JIT_EXIT_NUM = 0;
8436 ZEND_JIT_EXIT_COUNTERS = 0;
8437
8438 zend_jit_trace_init_caches();
8439 }
8440
