1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(zend_bool reattached)49 static int zend_jit_trace_startup(zend_bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 return FAILURE;
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 return FAILURE;
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 } else {
67 zend_jit_traces = ZCSG(jit_traces);
68 if (!zend_jit_traces) {
69 return FAILURE;
70 }
71 zend_jit_exit_groups = ZCSG(jit_exit_groups);
72 if (!zend_jit_exit_groups) {
73 return FAILURE;
74 }
75 }
76
77 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
78 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
79
80 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
81 if (JIT_G(exit_counters) == NULL) {
82 return FAILURE;
83 }
84
85 return SUCCESS;
86 }
87
zend_jit_trace_allocate_exit_group(uint32_t n)88 static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
89 {
90 dasm_State* dasm_state = NULL;
91 const void *entry;
92 char name[32];
93
94 dasm_init(&dasm_state, DASM_MAXSECTION);
95 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
96 dasm_setup(&dasm_state, dasm_actions);
97 zend_jit_trace_exit_group_stub(&dasm_state, n);
98
99 sprintf(name, "jit$$trace_exit_%d", n);
100 entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE);
101 dasm_free(&dasm_state);
102
103 #ifdef HAVE_DISASM
104 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
105 uint32_t i;
106
107 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
108 sprintf(name, "jit$$trace_exit_%d", n + i);
109 zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
110 }
111 }
112 #endif
113
114 return entry;
115 }
116
zend_jit_trace_allocate_exit_point(uint32_t n)117 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
118 {
119 const void *group = NULL;
120
121 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
122 return NULL;
123 }
124 do {
125 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
126 if (!group) {
127 return NULL;
128 }
129 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
130 group;
131 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
132 } while (n >= ZEND_JIT_EXIT_NUM);
133 return (const void*)
134 ((const char*)group +
135 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
136 }
137
zend_jit_trace_get_exit_addr(uint32_t n)138 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
139 {
140 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
141 return zend_jit_trace_allocate_exit_point(n);
142 }
143 return (const void*)
144 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
145 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
146 }
147
148 #if ZEND_JIT_TARGET_ARM64
zend_jit_get_current_trace_info(void)149 static zend_jit_trace_info *zend_jit_get_current_trace_info(void)
150 {
151 return &zend_jit_traces[ZEND_JIT_TRACE_NUM];
152 }
153
zend_jit_trace_find_exit_point(const void * addr)154 static uint32_t zend_jit_trace_find_exit_point(const void* addr)
155 {
156 uint32_t n = ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP;
157 uint32_t i;
158
159 for (i = 0; i < n; i++) {
160 if ((const char*)addr >= (const char*)zend_jit_exit_groups[i]
161 && (const char*)addr < (const char*)zend_jit_exit_groups[i] +
162 (ZEND_JIT_EXIT_POINTS_PER_GROUP * ZEND_JIT_EXIT_POINTS_SPACING)) {
163 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
164 ((const char*)addr - (const char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING;
165 }
166 }
167 return (uint32_t)-1;
168 }
169 #endif
170
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)171 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
172 {
173 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
174 uint32_t exit_point;
175 const zend_op_array *op_array;
176 uint32_t stack_offset = (uint32_t)-1;
177 uint32_t stack_size;
178 zend_jit_trace_stack *stack = NULL;
179
180 if (delayed_call_chain) {
181 assert(to_opline != NULL); /* CALL and IP share the same register */
182 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
183 }
184 if (JIT_G(current_frame)) {
185 op_array = &JIT_G(current_frame)->func->op_array;
186 stack_size = op_array->last_var + op_array->T;
187 if (stack_size) {
188 stack = JIT_G(current_frame)->stack;
189 do {
190 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
191 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
192 || STACK_REG(stack, stack_size-1) != ZREG_NONE) {
193 break;
194 }
195 stack_size--;
196 } while (stack_size);
197 }
198 } else {
199 op_array = NULL;
200 stack_size = 0;
201 }
202
203 /* Try to reuse exit points */
204 if (to_opline != NULL && t->exit_count > 0) {
205 uint32_t i = t->exit_count;
206
207 do {
208 i--;
209 if (stack_size == 0
210 || (t->exit_info[i].stack_size >= stack_size
211 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
212 stack_offset = t->exit_info[i].stack_offset;
213 if (t->exit_info[i].opline == to_opline
214 && t->exit_info[i].flags == flags
215 && t->exit_info[i].stack_size == stack_size) {
216 return i;
217 }
218 }
219 } while (i > 0);
220 }
221
222 exit_point = t->exit_count;
223 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
224 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
225 stack_offset = t->stack_map_size;
226 t->stack_map_size += stack_size;
227 // TODO: reduce number of reallocations ???
228 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
229 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
230 }
231 t->exit_count++;
232 t->exit_info[exit_point].opline = to_opline;
233 t->exit_info[exit_point].op_array = op_array;
234 t->exit_info[exit_point].flags = flags;
235 t->exit_info[exit_point].stack_size = stack_size;
236 t->exit_info[exit_point].stack_offset = stack_offset;
237 }
238
239 return exit_point;
240 }
241
zend_jit_trace_add_code(const void * start,uint32_t size)242 static void zend_jit_trace_add_code(const void *start, uint32_t size)
243 {
244 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
245
246 t->code_start = start;
247 t->code_size = size;
248 }
249
250 /**
251 * Locate a trace in the #zend_jit_traces array with the specified
252 * #code_start address.
253 *
254 * @return the #zend_jit_traces index or 0 if no such #code_start
255 * address was found
256 */
zend_jit_find_trace(const void * addr)257 static uint32_t zend_jit_find_trace(const void *addr)
258 {
259 uint32_t i;
260
261 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
262 if (zend_jit_traces[i].code_start == addr) {
263 return i;
264 }
265 }
266 return 0;
267 }
268
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)269 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
270 {
271 smart_str buf = {0};
272
273 smart_str_appends(&buf, TRACE_PREFIX);
274 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
275 smart_str_appendc(&buf, '$');
276 if (op_array->function_name) {
277 if (op_array->scope) {
278 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
279 smart_str_appends(&buf, "::");
280 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
281 } else {
282 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
283 }
284 } else if (op_array->filename) {
285 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
286 }
287 smart_str_appendc(&buf, '$');
288 smart_str_append_long(&buf, (zend_long)lineno);
289 smart_str_0(&buf);
290 return buf.s;
291 }
292
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)293 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
294 {
295 switch (opline->opcode) {
296 case ZEND_IS_IDENTICAL:
297 case ZEND_IS_NOT_IDENTICAL:
298 case ZEND_IS_EQUAL:
299 case ZEND_IS_NOT_EQUAL:
300 case ZEND_IS_SMALLER:
301 case ZEND_IS_SMALLER_OR_EQUAL:
302 case ZEND_CASE:
303 case ZEND_CASE_STRICT:
304 case ZEND_ISSET_ISEMPTY_CV:
305 case ZEND_ISSET_ISEMPTY_VAR:
306 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
307 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
308 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
309 case ZEND_INSTANCEOF:
310 case ZEND_TYPE_CHECK:
311 case ZEND_DEFINED:
312 case ZEND_IN_ARRAY:
313 case ZEND_ARRAY_KEY_EXISTS:
314 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
315 /* smart branch */
316 return 1;
317 }
318 break;
319 case ZEND_JMPZ:
320 case ZEND_JMPNZ:
321 case ZEND_JMPZ_EX:
322 case ZEND_JMPNZ_EX:
323 case ZEND_JMP_SET:
324 case ZEND_COALESCE:
325 case ZEND_JMP_NULL:
326 case ZEND_FE_RESET_R:
327 case ZEND_FE_RESET_RW:
328 case ZEND_ASSERT_CHECK:
329 case ZEND_FE_FETCH_R:
330 case ZEND_FE_FETCH_RW:
331 case ZEND_SWITCH_LONG:
332 case ZEND_SWITCH_STRING:
333 case ZEND_MATCH:
334 /* branch opcodes */
335 return 1;
336 case ZEND_NEW:
337 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
338 /* NEW may skip constructor without arguments */
339 return 1;
340 }
341 break;
342 case ZEND_CATCH:
343 case ZEND_FAST_CALL:
344 case ZEND_FAST_RET:
345 case ZEND_GENERATOR_CREATE:
346 case ZEND_GENERATOR_RETURN:
347 case ZEND_EXIT:
348 case ZEND_YIELD:
349 case ZEND_YIELD_FROM:
350 case ZEND_INCLUDE_OR_EVAL:
351 case ZEND_MATCH_ERROR:
352 /* unsupported */
353 return 1;
354 case ZEND_DO_FCALL:
355 /* potentially polymorphic call */
356 return 1;
357 #if 0
358 case ZEND_DO_UCALL:
359 case ZEND_DO_FCALL_BY_NAME:
360 /* monomorphic call */
361 // TODO: recompilation may change target ???
362 return 0;
363 #endif
364 case ZEND_RETURN_BY_REF:
365 case ZEND_RETURN:
366 /* return */
367 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
368 default:
369 break;
370 }
371 return 0;
372 }
373
zend_jit_trace_type_to_info_ex(zend_uchar type,uint32_t info)374 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(zend_uchar type, uint32_t info)
375 {
376 if (type == IS_UNKNOWN) {
377 return info;
378 }
379 ZEND_ASSERT(info & (1 << type));
380 if (type < IS_STRING) {
381 return (1 << type);
382 } else if (type != IS_ARRAY) {
383 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
384 } else {
385 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
386 }
387 }
388
zend_jit_trace_type_to_info(zend_uchar type)389 static zend_always_inline uint32_t zend_jit_trace_type_to_info(zend_uchar type)
390 {
391 return zend_jit_trace_type_to_info_ex(type, -1);
392 }
393
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)394 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
395 {
396 if (var >= op_array->last_var) {
397 return NO_ALIAS;
398 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
399 return SYMTABLE_ALIAS;
400 } else if (ssa->vars) {
401 return ssa->vars[var].alias;
402 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
403 return HTTP_RESPONSE_HEADER_ALIAS;
404 }
405 return NO_ALIAS;
406 }
407
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)408 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
409 int ssa_var,
410 uint8_t op_type)
411 {
412 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
413
414 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
415 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
416 info->type |= MAY_BE_GUARD;
417 } else {
418 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
419 }
420 }
421 }
422
423 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
424 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
425 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
426 } \
427 } while (0)
428
429 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
430 if (op_type != IS_UNKNOWN) { \
431 if ((op_info & MAY_BE_GUARD) != 0) { \
432 if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \
433 goto jit_failure; \
434 } \
435 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
436 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
437 op_info = zend_jit_trace_type_to_info(op_type); \
438 } else { \
439 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
440 op_info &= ~MAY_BE_GUARD; \
441 ssa->var_info[_ssa_var].type &= op_info; \
442 } \
443 } \
444 } \
445 } while (0)
446
447 #define ADD_OP1_TRACE_GUARD() \
448 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
449 #define ADD_OP2_TRACE_GUARD() \
450 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
451 #define ADD_OP1_DATA_TRACE_GUARD() \
452 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
453
454 #define CHECK_OP1_TRACE_TYPE() \
455 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
456 #define CHECK_OP2_TRACE_TYPE() \
457 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
458 #define CHECK_OP1_DATA_TRACE_TYPE() \
459 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
460
zend_jit_trace_frame_size(const zend_op_array * op_array)461 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
462 {
463 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
464 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
465 } else if (op_array) {
466 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
467 } else {
468 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
469 }
470 }
471
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)472 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
473 {
474 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
475 }
476
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)477 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
478 {
479 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
480 }
481
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,zend_uchar type)482 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, zend_uchar type)
483 {
484 zend_jit_trace_stack *stack = call->stack;
485 const zend_op_array *op_array = &call->func->op_array;
486 uint32_t arg_num = opline->op2.num;
487
488 if (arg_num > op_array->num_args) {
489 return;
490 }
491 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
492 zend_arg_info *arg_info;
493
494 ZEND_ASSERT(arg_num <= op_array->num_args);
495 arg_info = &op_array->arg_info[arg_num-1];
496
497 if (ZEND_TYPE_IS_SET(arg_info->type)) {
498 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
499 return;
500 }
501 }
502 }
503 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
504 }
505
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)506 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
507 {
508 if (func
509 && func->type == ZEND_INTERNAL_FUNCTION
510 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
511 && arg_num < func->internal_function.num_args) {
512 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
513
514 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
515 && ZEND_TYPE_IS_SET(arg_info->type)
516 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
517 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
518 && JIT_G(current_frame)
519 && JIT_G(current_frame)->call
520 && JIT_G(current_frame)->call->func) {
521 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
522
523 if (type != IS_UNKNOWN
524 && type < IS_STRING
525 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
526 return 0;
527 }
528 }
529 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
530 const zend_op *opline = call_info->arg_info[arg_num].opline;
531
532 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
533 zval *zv = RT_CONSTANT(opline, opline->op1);
534
535 if (!Z_REFCOUNTED_P(zv)) {
536 uint32_t type = Z_TYPE_P(zv);
537
538 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
539 if (type != IS_ARRAY
540 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
541 return 0;
542 }
543 }
544 }
545 }
546 }
547 }
548
549 return 1;
550 }
551
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)552 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
553 {
554 zend_jit_op_array_trace_extension *jit_extension;
555 zend_ssa *ssa;
556
557 jit_extension =
558 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
559 jit_extension->func_info.num = 0;
560 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
561 | ZEND_FUNC_JIT_ON_PROF_REQUEST
562 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
563 | ZEND_FUNC_JIT_ON_HOT_TRACE;
564 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
565 ssa = &jit_extension->func_info.ssa;
566
567 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
568 do {
569 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
570 break;
571 }
572
573 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
574 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info);
575 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
576 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
577 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
578 }
579 }
580
581 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
582 break;
583 }
584
585 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
586 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
587 }
588 return ssa;
589 } while (0);
590 }
591
592 memset(ssa, 0, sizeof(zend_ssa));
593 ssa->cfg.blocks_count = 1;
594
595 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
596 zend_cfg cfg;
597 void *checkpoint = zend_arena_checkpoint(CG(arena));
598
599 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
600 ssa->cfg.flags = cfg.flags;
601 } else{
602 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
603 }
604
605 /* TODO: move this to zend_cfg.c ? */
606 if (!op_array->function_name) {
607 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
608 }
609
610 zend_arena_release(&CG(arena), checkpoint);
611 }
612
613 return ssa;
614 }
615
616 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
617 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
618
zend_jit_trace_op_len(const zend_op * opline)619 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
620 {
621 int len;
622
623 switch (opline->opcode) {
624 case ZEND_ASSIGN_DIM:
625 case ZEND_ASSIGN_OBJ:
626 case ZEND_ASSIGN_STATIC_PROP:
627 case ZEND_ASSIGN_DIM_OP:
628 case ZEND_ASSIGN_OBJ_OP:
629 case ZEND_ASSIGN_STATIC_PROP_OP:
630 case ZEND_ASSIGN_OBJ_REF:
631 case ZEND_ASSIGN_STATIC_PROP_REF:
632 return 2; /* OP_DATA */
633 case ZEND_RECV_INIT:
634 len = 1;
635 opline++;
636 while (opline->opcode == ZEND_RECV_INIT) {
637 len++;
638 opline++;
639 }
640 return len;
641 case ZEND_BIND_GLOBAL:
642 len = 1;
643 opline++;
644 while (opline->opcode == ZEND_BIND_GLOBAL) {
645 len++;
646 opline++;
647 }
648 return len;
649 // case ZEND_IS_IDENTICAL:
650 // case ZEND_IS_NOT_IDENTICAL:
651 // case ZEND_IS_EQUAL:
652 // case ZEND_IS_NOT_EQUAL:
653 // case ZEND_IS_SMALLER:
654 // case ZEND_IS_SMALLER_OR_EQUAL:
655 // case ZEND_CASE:
656 // case ZEND_ISSET_ISEMPTY_CV:
657 // case ZEND_ISSET_ISEMPTY_VAR:
658 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
659 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
660 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
661 // case ZEND_INSTANCEOF:
662 // case ZEND_TYPE_CHECK:
663 // case ZEND_DEFINED:
664 // case ZEND_IN_ARRAY:
665 // case ZEND_ARRAY_KEY_EXISTS:
666 default:
667 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
668 return 2; /* JMPZ/JMPNZ */
669 }
670 return 1;
671 }
672 }
673
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)674 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
675 {
676 const zend_op_array *op_array;
677 zend_jit_trace_rec *p;
678 int k, vars_count;
679 zend_bitset use, def;
680 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
681 uint32_t set_size;
682 zend_ssa_phi *prev = NULL;
683 int level = 0;
684 ALLOCA_FLAG(use_heap);
685
686 op_array = trace_buffer->op_array;
687 set_size = zend_bitset_len(op_array->last_var + op_array->T);
688 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
689 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
690 def = use + set_size;
691 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
692 for (;;p++) {
693 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
694 const zend_op *opline = p->opline;
695 int len;
696
697 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
698 len = zend_jit_trace_op_len(opline);
699 while (len > 1) {
700 opline++;
701 if (opline->opcode != ZEND_OP_DATA) {
702 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
703 }
704 len--;
705 }
706 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
707 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
708 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
709 level++;
710 } else if (p->op == ZEND_JIT_TRACE_BACK) {
711 if (level == 0) {
712 // Phi for recursive calls and returns are not supported yet ???
713 assert(0);
714 } else {
715 level--;
716 }
717 } else if (p->op == ZEND_JIT_TRACE_END) {
718 break;
719 }
720 }
721
722 zend_bitset_intersection(use, def, set_size);
723
724 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
725 vars_count = op_array->last_var;
726 } else {
727 vars_count = op_array->last_var + op_array->T;
728 }
729 for (k = 0; k < vars_count; k++) {
730 if (zend_bitset_in(use, k)) {
731 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
732 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
733 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
734 sizeof(void*) * 2);
735 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
736 phi->sources[0] = STACK_VAR(stack, k);
737 phi->sources[1] = -1;
738 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
739 phi->pi = -1;
740 phi->var = k;
741 phi->ssa_var = ssa_vars_count;
742 SET_STACK_VAR(stack, k, ssa_vars_count);
743 ssa_vars_count++;
744 phi->block = 1;
745 if (prev) {
746 prev->next = phi;
747 } else {
748 tssa->blocks[1].phis = phi;
749 }
750 prev = phi;
751 }
752 }
753
754 free_alloca(use, use_heap);
755
756 return ssa_vars_count;
757 }
758
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)759 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
760 {
761 zend_ssa_phi *prev = NULL;
762 const zend_op_array *op_array = trace_buffer->op_array;
763 const zend_op *opline = trace_buffer[1].opline;
764 int count = opline - op_array->opcodes;
765 int i;
766
767 for(i = 0; i < count; i++) {
768 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
769 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
770 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
771 sizeof(void*) * 2);
772 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
773 phi->sources[0] = STACK_VAR(stack, i);
774 phi->sources[1] = -1;
775 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
776 phi->pi = -1;
777 phi->var = i;
778 phi->ssa_var = ssa_vars_count;
779 SET_STACK_VAR(stack, i, ssa_vars_count);
780 ssa_vars_count++;
781 phi->block = 1;
782 if (prev) {
783 prev->next = phi;
784 } else {
785 tssa->blocks[1].phis = phi;
786 }
787 prev = phi;
788 }
789 return ssa_vars_count;
790 }
791
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)792 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
793 {
794 const zend_op *opline = trace_buffer[1].opline - 1;
795 int i;
796
797 if (RETURN_VALUE_USED(opline)) {
798 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
799 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
800 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
801 sizeof(void*) * 2);
802
803 i = EX_VAR_TO_NUM(opline->result.var);
804 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
805 phi->sources[0] = STACK_VAR(stack, i);
806 phi->sources[1] = -1;
807 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
808 phi->pi = -1;
809 phi->var = i;
810 phi->ssa_var = ssa_vars_count;
811 SET_STACK_VAR(stack, i, ssa_vars_count);
812 ssa_vars_count++;
813 phi->block = 1;
814 tssa->blocks[1].phis = phi;
815 }
816 return ssa_vars_count;
817 }
818
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)819 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
820 {
821 int var, use;
822 zend_ssa_op *op;
823 zend_ssa_var_info *info;
824 unsigned int no_val;
825 zend_ssa_alias_kind alias;
826
827 if (tssa->vars[ssa_var].phi_use_chain) {
828 // TODO: this may be incorrect ???
829 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
830 } else {
831 var = ssa_var;
832 }
833 use = tssa->vars[var].use_chain;
834 if (use >= 0) {
835 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
836 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
837 if (tssa->ops[use].op1_use == var) {
838 no_val = ssa->vars[op->op1_use].no_val;
839 alias = ssa->vars[op->op1_use].alias;
840 info = ssa->var_info + op->op1_use;
841 } else if (tssa->ops[use].op2_use == var) {
842 no_val = ssa->vars[op->op2_use].no_val;
843 alias = ssa->vars[op->op2_use].alias;
844 info = ssa->var_info + op->op2_use;
845 } else if (tssa->ops[use].result_use == var) {
846 no_val = ssa->vars[op->result_use].no_val;
847 alias = ssa->vars[op->result_use].alias;
848 info = ssa->var_info + op->result_use;
849 } else {
850 assert(0);
851 return 0;
852 }
853 tssa->vars[ssa_var].no_val = no_val;
854 tssa->vars[ssa_var].alias = alias;
855 memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info));
856 return 1;
857 }
858 return 0;
859 }
860
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)861 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
862 {
863 zend_ssa_range tmp;
864 int def = tssa->vars[ssa_var].definition;
865
866 if (tssa->vars[ssa_var].alias == NO_ALIAS
867 && zend_inference_propagate_range(op_array, tssa, (zend_op*)tssa_opcodes[def], (zend_ssa_op*)&tssa->ops[def], ssa_var, &tmp)) {
868 tssa->var_info[ssa_var].range.min = tmp.min;
869 tssa->var_info[ssa_var].range.max = tmp.max;
870 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
871 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
872 tssa->var_info[ssa_var].has_range = 1;
873 }
874 }
875
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)876 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
877 {
878 int def;
879 zend_ssa_op *op;
880 zend_ssa_var_info *info;
881 unsigned int no_val;
882 zend_ssa_alias_kind alias;
883
884 def = tssa->vars[ssa_var].definition;
885 if (def >= 0) {
886 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
887 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
888 if (tssa->ops[def].op1_def == ssa_var) {
889 no_val = ssa->vars[op->op1_def].no_val;
890 alias = ssa->vars[op->op1_def].alias;
891 info = ssa->var_info + op->op1_def;
892 } else if (tssa->ops[def].op2_def == ssa_var) {
893 no_val = ssa->vars[op->op2_def].no_val;
894 alias = ssa->vars[op->op2_def].alias;
895 info = ssa->var_info + op->op2_def;
896 } else if (tssa->ops[def].result_def == ssa_var) {
897 no_val = ssa->vars[op->result_def].no_val;
898 alias = ssa->vars[op->result_def].alias;
899 info = ssa->var_info + op->result_def;
900 } else {
901 assert(0);
902 return;
903 }
904
905 tssa->vars[ssa_var].no_val = no_val;
906 tssa->vars[ssa_var].alias = alias;
907
908 if (!(info->type & MAY_BE_REF)) {
909 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
910 }
911
912 if (info->has_range) {
913 if (tssa->var_info[ssa_var].has_range) {
914 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
915 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
916 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
917 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
918 } else {
919 tssa->var_info[ssa_var].has_range = 1;
920 tssa->var_info[ssa_var].range = info->range;
921 }
922 }
923 }
924 }
925
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)926 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
927 {
928 int def;
929 zend_ssa_op *op;
930 zend_ssa_var_info *info;
931
932 def = tssa->vars[ssa_var].definition;
933 if (def >= 0) {
934 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
935 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
936 if (tssa->ops[def].op1_def == ssa_var) {
937 info = ssa->var_info + op->op1_def;
938 } else if (tssa->ops[def].op2_def == ssa_var) {
939 info = ssa->var_info + op->op2_def;
940 } else if (tssa->ops[def].result_def == ssa_var) {
941 info = ssa->var_info + op->result_def;
942 } else {
943 assert(0);
944 return 0;
945 }
946 tssa->var_info[ssa_var].type &= info->type;
947 if (info->ce) {
948 if (tssa->var_info[ssa_var].ce) {
949 if (tssa->var_info[ssa_var].ce != info->ce) {
950 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
951 /* everything fine */
952 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
953 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
954 } else {
955 // TODO: classes may implement the same interface ???
956 //ZEND_UNREACHABLE();
957 }
958 }
959 tssa->var_info[ssa_var].is_instanceof =
960 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
961 } else {
962 tssa->var_info[ssa_var].ce = info->ce;
963 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
964 }
965 }
966 if (info->has_range) {
967 if (tssa->var_info[ssa_var].has_range) {
968 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
969 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
970 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
971 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
972 } else {
973 tssa->var_info[ssa_var].has_range = 1;
974 tssa->var_info[ssa_var].range = info->range;
975 }
976 }
977 return 1;
978 }
979 return 0;
980 }
981
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)982 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
983 {
984 while (1) {
985 if (p->op == ZEND_JIT_TRACE_VM) {
986 if (p->opline->opcode == ZEND_DO_UCALL
987 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
988 || p->opline->opcode == ZEND_DO_FCALL) {
989 if (p->opline->result_type != IS_UNUSED) {
990 return ssa_op->result_def;
991 }
992 }
993 return -1;
994 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
995 /*skip */
996 } else {
997 return -1;
998 }
999 p--;
1000 }
1001 }
1002
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1003 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1004 {
1005 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1006 p--;
1007 while (1) {
1008 if (p->op == ZEND_JIT_TRACE_VM) {
1009 if (p->opline->opcode == ZEND_INIT_FCALL
1010 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1011 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1012 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1013 || p->opline->opcode == ZEND_INIT_USER_CALL
1014 || p->opline->opcode == ZEND_NEW
1015 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1016 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
1017 return p->opline;
1018 }
1019 return NULL;
1020 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1021 /*skip */
1022 } else {
1023 return NULL;
1024 }
1025 p--;
1026 }
1027 } else {
1028 const zend_op *opline = NULL;
1029 int call_level = 0;
1030
1031 p++;
1032 while (1) {
1033 if (p->op == ZEND_JIT_TRACE_VM) {
1034 opline = p->opline;
1035 break;
1036 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1037 call_level++;
1038 /*skip */
1039 } else {
1040 return NULL;
1041 }
1042 p--;
1043 }
1044 if (opline) {
1045 while (opline > op_array->opcodes) {
1046 opline--;
1047 switch (opline->opcode) {
1048 case ZEND_INIT_FCALL:
1049 case ZEND_INIT_FCALL_BY_NAME:
1050 case ZEND_INIT_NS_FCALL_BY_NAME:
1051 case ZEND_INIT_METHOD_CALL:
1052 case ZEND_INIT_DYNAMIC_CALL:
1053 case ZEND_INIT_STATIC_METHOD_CALL:
1054 case ZEND_INIT_USER_CALL:
1055 case ZEND_NEW:
1056 if (call_level == 0) {
1057 return opline;
1058 }
1059 call_level--;
1060 break;
1061 case ZEND_DO_FCALL:
1062 case ZEND_DO_ICALL:
1063 case ZEND_DO_UCALL:
1064 case ZEND_DO_FCALL_BY_NAME:
1065 case ZEND_CALLABLE_CONVERT:
1066 call_level++;
1067 break;
1068 }
1069 }
1070 }
1071 }
1072 return NULL;
1073 }
1074
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1075 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1076 {
1077 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1078 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1079 int idx = tssa->vars[var].definition;
1080
1081 if (idx >= 0) {
1082 if (tssa->ops[idx].op1_def == var) {
1083 const zend_op *opline = ssa_opcodes[idx];
1084 if (opline->opcode == ZEND_PRE_DEC
1085 || opline->opcode == ZEND_PRE_INC
1086 || opline->opcode == ZEND_POST_DEC
1087 || opline->opcode == ZEND_POST_INC) {
1088 if (tssa->ops[idx].op1_use >= 0
1089 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1090 return 0;
1091 }
1092 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1093 return 0;
1094 }
1095 return 1;
1096 } else if (opline->opcode == ZEND_ASSIGN_OP
1097 && (opline->extended_value == ZEND_ADD
1098 || opline->extended_value == ZEND_SUB
1099 || opline->extended_value == ZEND_MUL)) {
1100 if ((opline->op2_type & (IS_VAR|IS_CV))
1101 && tssa->ops[idx].op2_use >= 0
1102 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1103 return 0;
1104 }
1105 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1106 return 0;
1107 }
1108 if (opline->op2_type == IS_CONST) {
1109 zval *zv = RT_CONSTANT(opline, opline->op2);
1110 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1111 return 0;
1112 }
1113 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1114 return 0;
1115 }
1116 return 1;
1117 }
1118 }
1119 if (tssa->ops[idx].result_def == var) {
1120 const zend_op *opline = ssa_opcodes[idx];
1121 if (opline->opcode == ZEND_ADD
1122 || opline->opcode == ZEND_SUB
1123 || opline->opcode == ZEND_MUL) {
1124 if ((opline->op1_type & (IS_VAR|IS_CV))
1125 && tssa->ops[idx].op1_use >= 0
1126 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1127 return 0;
1128 }
1129 if ((opline->op2_type & (IS_VAR|IS_CV))
1130 && tssa->ops[idx].op2_use >= 0
1131 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1132 return 0;
1133 }
1134 if (opline->op1_type == IS_CONST) {
1135 zval *zv = RT_CONSTANT(opline, opline->op1);
1136 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1137 return 0;
1138 }
1139 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1140 return 0;
1141 }
1142 if (opline->op2_type == IS_CONST) {
1143 zval *zv = RT_CONSTANT(opline, opline->op2);
1144 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1145 return 0;
1146 }
1147 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1148 return 0;
1149 }
1150 } else if (opline->opcode == ZEND_PRE_DEC
1151 || opline->opcode == ZEND_PRE_INC
1152 || opline->opcode == ZEND_POST_DEC
1153 || opline->opcode == ZEND_POST_INC) {
1154 if ((opline->op1_type & (IS_VAR|IS_CV))
1155 && tssa->ops[idx].op1_use >= 0
1156 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1157 return 0;
1158 }
1159 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1160 return 0;
1161 }
1162 return 1;
1163 }
1164 }
1165 }
1166 }
1167 return 0;
1168 }
1169
1170 typedef struct _zend_tssa {
1171 zend_ssa ssa;
1172 const zend_op **tssa_opcodes;
1173 int used_stack;
1174 } zend_tssa;
1175
1176 static const zend_op _nop_opcode = {0};
1177
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1178 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1179 {
1180 zend_ssa *tssa;
1181 zend_ssa_op *ssa_ops, *op;
1182 zend_ssa_var *ssa_vars;
1183 zend_ssa_var_info *ssa_var_info;
1184 const zend_op_array *op_array;
1185 const zend_op *opline;
1186 const zend_op **ssa_opcodes;
1187 zend_jit_trace_rec *p;
1188 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1189 zend_jit_trace_stack *stack;
1190 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1191 uint32_t optimization_level = 0;
1192 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1193 size_t frame_size, stack_top, stack_size, stack_bottom;
1194 zend_jit_op_array_trace_extension *jit_extension;
1195 zend_ssa *ssa;
1196 zend_jit_trace_stack_frame *frame, *top, *call;
1197 zend_ssa_var_info return_value_info;
1198
1199 /* 1. Count number of TSSA opcodes;
1200 * Count number of activation frames;
1201 * Calculate size of abstract stack;
1202 * Construct regular SSA for involved op_array */
1203 op_array = trace_buffer->op_array;
1204 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1205 stack_bottom = 0;
1206 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1207 ssa_ops_count = 0;
1208 call_level = 0;
1209 level = 0;
1210 num_op_arrays = 0;
1211 /* Remember op_array to cleanup */
1212 op_arrays[num_op_arrays++] = op_array;
1213 /* Build SSA */
1214 ssa = zend_jit_trace_build_ssa(op_array, script);
1215 for (;;p++) {
1216 if (p->op == ZEND_JIT_TRACE_VM) {
1217 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1218 const zend_op *opline = p->opline;
1219
1220 switch (opline->opcode) {
1221 case ZEND_INCLUDE_OR_EVAL:
1222 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1223 break;
1224 case ZEND_FETCH_R:
1225 case ZEND_FETCH_W:
1226 case ZEND_FETCH_RW:
1227 case ZEND_FETCH_FUNC_ARG:
1228 case ZEND_FETCH_IS:
1229 case ZEND_FETCH_UNSET:
1230 case ZEND_UNSET_VAR:
1231 case ZEND_ISSET_ISEMPTY_VAR:
1232 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1233 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1234 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1235 !op_array->function_name) {
1236 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1237 }
1238 break;
1239 }
1240 }
1241 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1242 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1243 call_level++;
1244 stack_top += zend_jit_trace_frame_size(p->op_array);
1245 if (stack_top > stack_size) {
1246 stack_size = stack_top;
1247 }
1248 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1249 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1250 if (p->func
1251 && p->func != (zend_function*)&zend_pass_function
1252 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1253 || zend_string_equals_literal(p->func->common.function_name, "compact")
1254 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1255 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1256 }
1257 }
1258 frame_size = zend_jit_trace_frame_size(p->op_array);
1259 if (call_level == 0) {
1260 if (stack_top + frame_size > stack_size) {
1261 stack_size = stack_top + frame_size;
1262 }
1263 } else {
1264 call_level--;
1265 stack_top -= frame_size;
1266 }
1267 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1268 op_array = p->op_array;
1269 if (call_level == 0) {
1270 stack_top += zend_jit_trace_frame_size(op_array);
1271 if (stack_top > stack_size) {
1272 stack_size = stack_top;
1273 }
1274 } else {
1275 call_level--;
1276 }
1277 level++;
1278 jit_extension =
1279 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1280 ssa = &jit_extension->func_info.ssa;
1281 if (ssa->cfg.blocks_count) {
1282 /* pass */
1283 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1284 /* Too many functions in single trace */
1285 *num_op_arrays_ptr = num_op_arrays;
1286 return NULL;
1287 } else {
1288 /* Remember op_array to cleanup */
1289 op_arrays[num_op_arrays++] = op_array;
1290 /* Build SSA */
1291 ssa = zend_jit_trace_build_ssa(op_array, script);
1292 }
1293 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1294 if (level == 0) {
1295 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1296 jit_extension =
1297 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1298 ssa = &jit_extension->func_info.ssa;
1299 if (ssa->cfg.blocks_count) {
1300 /* pass */
1301 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1302 /* Too many functions in single trace */
1303 *num_op_arrays_ptr = num_op_arrays;
1304 return NULL;
1305 } else {
1306 /* Remember op_array to cleanup */
1307 op_arrays[num_op_arrays++] = op_array;
1308 /* Build SSA */
1309 ssa = zend_jit_trace_build_ssa(op_array, script);
1310 }
1311 } else {
1312 stack_top -= zend_jit_trace_frame_size(op_array);
1313 level--;
1314 }
1315 op_array = p->op_array;
1316 } else if (p->op == ZEND_JIT_TRACE_END) {
1317 break;
1318 }
1319 }
1320 *num_op_arrays_ptr = num_op_arrays;
1321
1322 /* Allocate space for abstract stack */
1323 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1324
1325 /* 2. Construct TSSA */
1326 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1327 tssa->cfg.flags = ZEND_SSA_TSSA;
1328 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1329 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1330 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1331
1332 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1333 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1334 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1335 tssa->cfg.blocks_count = 2;
1336 tssa->cfg.edges_count = 2;
1337
1338 tssa->cfg.predecessors[0] = 0;
1339 tssa->cfg.predecessors[1] = 1;
1340
1341 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1342 tssa->cfg.blocks[0].successors_count = 1;
1343 tssa->cfg.blocks[0].predecessors_count = 0;
1344 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1345 tssa->cfg.blocks[0].successors[0] = 1;
1346
1347 tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1348 tssa->cfg.blocks[1].successors_count = 1;
1349 tssa->cfg.blocks[1].predecessors_count = 2;
1350 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1351 tssa->cfg.blocks[1].successors[1] = 1;
1352 } else {
1353 tssa->cfg.blocks_count = 1;
1354 tssa->cfg.edges_count = 0;
1355
1356 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1357 tssa->cfg.blocks[0].successors_count = 0;
1358 tssa->cfg.blocks[0].predecessors_count = 0;
1359 }
1360 ((zend_tssa*)tssa)->used_stack = -1;
1361
1362 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1363 return tssa;
1364 }
1365
1366 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1367 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1368 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1369 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1370 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1371
1372 op_array = trace_buffer->op_array;
1373 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1374 ssa_vars_count = op_array->last_var;
1375 } else {
1376 ssa_vars_count = op_array->last_var + op_array->T;
1377 }
1378 stack = frame->stack;
1379 for (i = 0; i < ssa_vars_count; i++) {
1380 SET_STACK_VAR(stack, i, i);
1381 }
1382
1383 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1384 // TODO: For tracing, it's possible, to create pseudo Phi functions
1385 // at the end of loop, without this additional pass (like LuaJIT) ???
1386 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1387 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1388 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1389 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1390 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1391 }
1392
1393 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1394 idx = 0;
1395 level = 0;
1396 for (;;p++) {
1397 if (p->op == ZEND_JIT_TRACE_VM) {
1398 opline = p->opline;
1399 ssa_opcodes[idx] = opline;
1400 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1401 idx++;
1402 len = zend_jit_trace_op_len(p->opline);
1403 while (len > 1) {
1404 opline++;
1405 ssa_opcodes[idx] = opline;
1406 if (opline->opcode != ZEND_OP_DATA) {
1407 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1408 }
1409 idx++;
1410 len--;
1411 }
1412 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1413 frame = zend_jit_trace_call_frame(frame, op_array);
1414 stack = frame->stack;
1415 op_array = p->op_array;
1416 level++;
1417 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1418 return NULL;
1419 }
1420 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1421 for (i = 0; i < op_array->last_var; i++) {
1422 SET_STACK_VAR(stack, i, ssa_vars_count++);
1423 }
1424 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1425 op_array = p->op_array;
1426 frame = zend_jit_trace_ret_frame(frame, op_array);
1427 stack = frame->stack;
1428 if (level == 0) {
1429 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1430 return NULL;
1431 }
1432 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1433 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1434 SET_STACK_VAR(stack, i, ssa_vars_count++);
1435 }
1436 } else {
1437 level--;
1438 }
1439 } else if (p->op == ZEND_JIT_TRACE_END) {
1440 break;
1441 }
1442 }
1443
1444 op_array = trace_buffer->op_array;
1445 tssa->vars_count = ssa_vars_count;
1446 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1447 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1448 vars_count = op_array->last_var;
1449 } else {
1450 vars_count = op_array->last_var + op_array->T;
1451 }
1452 i = 0;
1453 while (i < vars_count) {
1454 ssa_vars[i].var = i;
1455 ssa_vars[i].scc = -1;
1456 ssa_vars[i].definition = -1;
1457 ssa_vars[i].use_chain = -1;
1458 i++;
1459 }
1460 while (i < tssa->vars_count) {
1461 ssa_vars[i].var = -1;
1462 ssa_vars[i].scc = -1;
1463 ssa_vars[i].definition = -1;
1464 ssa_vars[i].use_chain = -1;
1465 i++;
1466 }
1467
1468 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1469 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1470 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1471 /* Update Phi sources */
1472 zend_ssa_phi *phi = tssa->blocks[1].phis;
1473
1474 while (phi) {
1475 phi->sources[1] = STACK_VAR(stack, phi->var);
1476 ssa_vars[phi->ssa_var].var = phi->var;
1477 ssa_vars[phi->ssa_var].definition_phi = phi;
1478 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1479 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1480 phi = phi->next;
1481 }
1482 }
1483
1484 /* 3. Compute use-def chains */
1485 idx = (ssa_ops_count - 1);
1486 op = ssa_ops + idx;
1487 while (idx >= 0) {
1488 opline = ssa_opcodes[idx];
1489 if (op->op1_use >= 0) {
1490 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1491 ssa_vars[op->op1_use].use_chain = idx;
1492 }
1493 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1494 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1495 ssa_vars[op->op2_use].use_chain = idx;
1496 }
1497 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1498 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1499 ssa_vars[op->result_use].use_chain = idx;
1500 }
1501 if (op->op1_def >= 0) {
1502 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1503 ssa_vars[op->op1_def].definition = idx;
1504 }
1505 if (op->op2_def >= 0) {
1506 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1507 ssa_vars[op->op2_def].definition = idx;
1508 }
1509 if (op->result_def >= 0) {
1510 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1511 ssa_vars[op->result_def].definition = idx;
1512 }
1513 op--;
1514 idx--;
1515 }
1516
1517 /* 4. Type inference */
1518 op_array = trace_buffer->op_array;
1519 jit_extension =
1520 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1521 ssa = &jit_extension->func_info.ssa;
1522
1523 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1524
1525 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1526 i = 0;
1527 while (i < op_array->last_var) {
1528 if (i < op_array->num_args) {
1529 if (ssa->var_info
1530 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1531 /* pass */
1532 } else {
1533 if (ssa->vars) {
1534 ssa_vars[i].no_val = ssa->vars[i].no_val;
1535 ssa_vars[i].alias = ssa->vars[i].alias;
1536 } else {
1537 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1538 }
1539 if (op_array->arg_info) {
1540 zend_arg_info *arg_info = &op_array->arg_info[i];
1541 zend_class_entry *ce;
1542 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1543
1544 if (ZEND_ARG_SEND_MODE(arg_info)) {
1545 tmp |= MAY_BE_REF;
1546 }
1547 ssa_var_info[i].type = tmp;
1548 ssa_var_info[i].ce = ce;
1549 ssa_var_info[i].is_instanceof = 1;
1550 } else {
1551 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1552 }
1553 }
1554 } else {
1555 if (ssa->vars) {
1556 ssa_vars[i].no_val = ssa->vars[i].no_val;
1557 ssa_vars[i].alias = ssa->vars[i].alias;
1558 } else {
1559 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1560 }
1561 if (ssa_vars[i].alias == NO_ALIAS) {
1562 ssa_var_info[i].type = MAY_BE_UNDEF;
1563 } else {
1564 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1565 }
1566 }
1567 i++;
1568 }
1569 } else {
1570 int parent_vars_count = 0;
1571 zend_jit_trace_stack *parent_stack = NULL;
1572
1573 i = 0;
1574 if (parent_trace) {
1575 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1576 op_array->last_var + op_array->T);
1577 if (parent_vars_count) {
1578 parent_stack =
1579 zend_jit_traces[parent_trace].stack_map +
1580 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1581 }
1582 }
1583 while (i < op_array->last_var + op_array->T) {
1584 if (!ssa->var_info
1585 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1586 if (ssa->vars && i < ssa->vars_count) {
1587 ssa_vars[i].alias = ssa->vars[i].alias;
1588 } else {
1589 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1590 }
1591 if (i < op_array->last_var) {
1592 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1593 } else {
1594 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1595 }
1596 }
1597 if (i < parent_vars_count) {
1598 /* Initialize TSSA variable from parent trace */
1599 zend_uchar op_type = STACK_TYPE(parent_stack, i);
1600
1601 if (op_type != IS_UNKNOWN) {
1602 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1603 if (!ssa_var_info[i].type
1604 && op_type == IS_UNDEF
1605 && i >= op_array->last_var) {
1606 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1607 ssa_var_info[i].type |= MAY_BE_UNDEF;
1608 }
1609 }
1610 }
1611 i++;
1612 }
1613 }
1614
1615 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1616 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1617 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1618 /* Propagate initial value through Phi functions */
1619 zend_ssa_phi *phi = tssa->blocks[1].phis;
1620
1621 while (phi) {
1622 if (!ssa->var_info
1623 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) {
1624 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1625 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1626 }
1627 phi = phi->next;
1628 }
1629 }
1630
1631 frame = JIT_G(current_frame);
1632 top = zend_jit_trace_call_frame(frame, op_array);
1633 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1634 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1635 frame->used_stack = 0;
1636 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1637 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
1638 }
1639 memset(&return_value_info, 0, sizeof(return_value_info));
1640
1641 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1642 max_used_stack = used_stack = 0;
1643 } else {
1644 max_used_stack = used_stack = -1;
1645 }
1646
1647 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1648 idx = 0;
1649 level = 0;
1650 opline = NULL;
1651 for (;;p++) {
1652 if (p->op == ZEND_JIT_TRACE_VM) {
1653 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1654 uint8_t val_type = IS_UNKNOWN;
1655 // zend_class_entry *op1_ce = NULL;
1656 zend_class_entry *op2_ce = NULL;
1657
1658 opline = p->opline;
1659
1660 op1_type = orig_op1_type = p->op1_type;
1661 op2_type = orig_op2_type = p->op2_type;
1662 op3_type = p->op3_type;
1663 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1664 op1_type = IS_UNKNOWN;
1665 }
1666 if (op1_type != IS_UNKNOWN) {
1667 op1_type &= ~IS_TRACE_PACKED;
1668 }
1669 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1670 op2_type = IS_UNKNOWN;
1671 }
1672 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1673 op3_type = IS_UNKNOWN;
1674 }
1675
1676 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1677 // op1_ce = (zend_class_entry*)(p+1)->ce;
1678 p++;
1679 }
1680 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1681 op2_ce = (zend_class_entry*)(p+1)->ce;
1682 p++;
1683 }
1684 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1685 val_type = (p+1)->op1_type;
1686 p++;
1687 }
1688
1689 switch (opline->opcode) {
1690 case ZEND_ASSIGN_OP:
1691 if (opline->extended_value == ZEND_POW
1692 || opline->extended_value == ZEND_DIV) {
1693 // TODO: check for division by zero ???
1694 break;
1695 }
1696 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1697 break;
1698 }
1699 ADD_OP1_TRACE_GUARD();
1700 ADD_OP2_TRACE_GUARD();
1701 break;
1702 case ZEND_ASSIGN_DIM_OP:
1703 if (opline->extended_value == ZEND_POW
1704 || opline->extended_value == ZEND_DIV) {
1705 // TODO: check for division by zero ???
1706 break;
1707 }
1708 if (opline->result_type != IS_UNUSED) {
1709 break;
1710 }
1711 if (op3_type != IS_UNKNOWN
1712 && !zend_jit_supported_binary_op(
1713 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1714 break;
1715 }
1716 ZEND_FALLTHROUGH;
1717 case ZEND_ASSIGN_DIM:
1718 if (opline->op1_type == IS_CV) {
1719 if ((opline+1)->op1_type == IS_CV
1720 && (opline+1)->op1.var == opline->op1.var) {
1721 /* skip $a[x] = $a; */
1722 break;
1723 }
1724 ADD_OP1_DATA_TRACE_GUARD();
1725 ADD_OP2_TRACE_GUARD();
1726 ADD_OP1_TRACE_GUARD();
1727 } else if (orig_op1_type != IS_UNKNOWN
1728 && (orig_op1_type & IS_TRACE_INDIRECT)
1729 && opline->result_type == IS_UNUSED) {
1730 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1731 ADD_OP1_DATA_TRACE_GUARD();
1732 }
1733 ADD_OP2_TRACE_GUARD();
1734 }
1735 if (op1_type == IS_ARRAY
1736 && ((opline->op2_type == IS_CONST
1737 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1738 || (opline->op2_type != IS_CONST
1739 && op2_type == IS_LONG))) {
1740
1741 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1742 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1743
1744 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1745 info->type |= MAY_BE_PACKED_GUARD;
1746 info->type &= ~MAY_BE_ARRAY_PACKED;
1747 }
1748 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1749 && val_type != IS_UNKNOWN
1750 && val_type != IS_UNDEF) {
1751 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1752
1753 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1754 info->type |= MAY_BE_PACKED_GUARD;
1755 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1756 }
1757 }
1758 }
1759 break;
1760 case ZEND_ASSIGN_OBJ_OP:
1761 if (opline->extended_value == ZEND_POW
1762 || opline->extended_value == ZEND_DIV) {
1763 // TODO: check for division by zero ???
1764 break;
1765 }
1766 if (opline->result_type != IS_UNUSED) {
1767 break;
1768 }
1769 ZEND_FALLTHROUGH;
1770 case ZEND_ASSIGN_OBJ:
1771 case ZEND_PRE_INC_OBJ:
1772 case ZEND_PRE_DEC_OBJ:
1773 case ZEND_POST_INC_OBJ:
1774 case ZEND_POST_DEC_OBJ:
1775 if (opline->op2_type != IS_CONST
1776 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1777 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1778 break;
1779 }
1780 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1781 if (opline->op1_type == IS_CV
1782 && (opline+1)->op1_type == IS_CV
1783 && (opline+1)->op1.var == opline->op1.var) {
1784 /* skip $a->prop += $a; */
1785 break;
1786 }
1787 ADD_OP1_DATA_TRACE_GUARD();
1788 }
1789 ADD_OP1_TRACE_GUARD();
1790 break;
1791 case ZEND_CONCAT:
1792 case ZEND_FAST_CONCAT:
1793 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1794 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1795 ADD_OP2_TRACE_GUARD();
1796 ADD_OP1_TRACE_GUARD();
1797 }
1798 break;
1799 case ZEND_IS_EQUAL:
1800 case ZEND_IS_NOT_EQUAL:
1801 case ZEND_IS_SMALLER:
1802 case ZEND_IS_SMALLER_OR_EQUAL:
1803 case ZEND_CASE:
1804 case ZEND_IS_IDENTICAL:
1805 case ZEND_IS_NOT_IDENTICAL:
1806 case ZEND_CASE_STRICT:
1807 case ZEND_BW_OR:
1808 case ZEND_BW_AND:
1809 case ZEND_BW_XOR:
1810 case ZEND_SL:
1811 case ZEND_SR:
1812 case ZEND_MOD:
1813 case ZEND_ADD:
1814 case ZEND_SUB:
1815 case ZEND_MUL:
1816 // case ZEND_DIV: // TODO: check for division by zero ???
1817 ADD_OP2_TRACE_GUARD();
1818 ZEND_FALLTHROUGH;
1819 case ZEND_ECHO:
1820 case ZEND_STRLEN:
1821 case ZEND_COUNT:
1822 case ZEND_QM_ASSIGN:
1823 case ZEND_FE_RESET_R:
1824 ADD_OP1_TRACE_GUARD();
1825 break;
1826 case ZEND_FE_FETCH_R:
1827 ADD_OP1_TRACE_GUARD();
1828 if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) {
1829
1830 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1831
1832 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1833 info->type |= MAY_BE_PACKED_GUARD;
1834 if (orig_op1_type & IS_TRACE_PACKED) {
1835 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1836 } else {
1837 info->type &= ~MAY_BE_ARRAY_PACKED;
1838 }
1839 }
1840 }
1841 break;
1842 case ZEND_VERIFY_RETURN_TYPE:
1843 if (opline->op1_type == IS_UNUSED) {
1844 /* Always throws */
1845 break;
1846 }
1847 if (opline->op1_type == IS_CONST) {
1848 /* TODO Different instruction format, has return value */
1849 break;
1850 }
1851 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1852 /* Not worth bothering with */
1853 break;
1854 }
1855 ADD_OP1_TRACE_GUARD();
1856 break;
1857 case ZEND_FETCH_DIM_FUNC_ARG:
1858 if (!frame
1859 || !frame->call
1860 || !frame->call->func
1861 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1862 break;
1863 }
1864 ADD_OP2_TRACE_GUARD();
1865 ADD_OP1_TRACE_GUARD();
1866 break;
1867 case ZEND_PRE_INC:
1868 case ZEND_PRE_DEC:
1869 case ZEND_POST_INC:
1870 case ZEND_POST_DEC:
1871 if (opline->op1_type != IS_CV) {
1872 break;
1873 }
1874 ADD_OP1_TRACE_GUARD();
1875 break;
1876 case ZEND_ASSIGN:
1877 if (opline->op1_type != IS_CV) {
1878 break;
1879 }
1880 ADD_OP2_TRACE_GUARD();
1881 if (op1_type != IS_UNKNOWN
1882 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1883 ADD_OP1_TRACE_GUARD();
1884 }
1885 break;
1886 case ZEND_CAST:
1887 if (opline->extended_value != op1_type) {
1888 break;
1889 }
1890 ADD_OP1_TRACE_GUARD();
1891 break;
1892 case ZEND_JMPZ:
1893 case ZEND_JMPNZ:
1894 case ZEND_JMPZ_EX:
1895 case ZEND_JMPNZ_EX:
1896 case ZEND_BOOL:
1897 case ZEND_BOOL_NOT:
1898 ADD_OP1_TRACE_GUARD();
1899 break;
1900 case ZEND_ISSET_ISEMPTY_CV:
1901 if ((opline->extended_value & ZEND_ISEMPTY)) {
1902 // TODO: support for empty() ???
1903 break;
1904 }
1905 ADD_OP1_TRACE_GUARD();
1906 break;
1907 case ZEND_IN_ARRAY:
1908 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
1909 break;
1910 }
1911 ADD_OP1_TRACE_GUARD();
1912 break;
1913 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
1914 if ((opline->extended_value & ZEND_ISEMPTY)) {
1915 // TODO: support for empty() ???
1916 break;
1917 }
1918 ZEND_FALLTHROUGH;
1919 case ZEND_FETCH_DIM_R:
1920 case ZEND_FETCH_DIM_IS:
1921 case ZEND_FETCH_LIST_R:
1922 ADD_OP1_TRACE_GUARD();
1923 ADD_OP2_TRACE_GUARD();
1924
1925 if (op1_type == IS_ARRAY
1926 && opline->op1_type != IS_CONST
1927 && ((opline->op2_type == IS_CONST
1928 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1929 || (opline->op2_type != IS_CONST
1930 && op2_type == IS_LONG))) {
1931
1932 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1933
1934 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1935 info->type |= MAY_BE_PACKED_GUARD;
1936 if (orig_op1_type & IS_TRACE_PACKED) {
1937 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1938 } else {
1939 info->type &= ~MAY_BE_ARRAY_PACKED;
1940 }
1941 }
1942 }
1943 break;
1944 case ZEND_FETCH_DIM_W:
1945 case ZEND_FETCH_DIM_RW:
1946 // case ZEND_FETCH_DIM_UNSET:
1947 case ZEND_FETCH_LIST_W:
1948 if (opline->op1_type != IS_CV
1949 && (orig_op1_type == IS_UNKNOWN
1950 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
1951 break;
1952 }
1953 ADD_OP1_TRACE_GUARD();
1954 ADD_OP2_TRACE_GUARD();
1955 if (op1_type == IS_ARRAY
1956 && !(orig_op1_type & IS_TRACE_PACKED)
1957 && ((opline->op2_type == IS_CONST
1958 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1959 || (opline->op2_type != IS_CONST
1960 && op2_type == IS_LONG))) {
1961
1962 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1963
1964 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1965 info->type |= MAY_BE_PACKED_GUARD;
1966 info->type &= ~MAY_BE_ARRAY_PACKED;
1967 }
1968 }
1969 break;
1970 case ZEND_SEND_VAL_EX:
1971 case ZEND_SEND_VAR_EX:
1972 case ZEND_SEND_VAR_NO_REF_EX:
1973 if (opline->op2_type == IS_CONST) {
1974 /* Named parameters not supported in JIT */
1975 break;
1976 }
1977 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
1978 goto propagate_arg;
1979 }
1980 ZEND_FALLTHROUGH;
1981 case ZEND_SEND_VAL:
1982 case ZEND_SEND_VAR:
1983 case ZEND_SEND_VAR_NO_REF:
1984 case ZEND_SEND_FUNC_ARG:
1985 if (opline->op2_type == IS_CONST) {
1986 /* Named parameters not supported in JIT */
1987 break;
1988 }
1989 ADD_OP1_TRACE_GUARD();
1990 propagate_arg:
1991 /* Propagate argument type */
1992 if (frame->call
1993 && frame->call->func
1994 && frame->call->func->type == ZEND_USER_FUNCTION
1995 && opline->op2.num <= frame->call->func->op_array.num_args) {
1996 uint32_t info;
1997
1998 if (opline->op1_type == IS_CONST) {
1999 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
2000 } else {
2001 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2002 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
2003 }
2004 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
2005 zend_arg_info *arg_info;
2006
2007 ZEND_ASSERT(frame->call->func->op_array.arg_info);
2008 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
2009 if (ZEND_TYPE_IS_SET(arg_info->type)) {
2010 zend_class_entry *ce;
2011 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2012 info &= tmp;
2013 if (!info) {
2014 break;
2015 }
2016 }
2017 }
2018 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2019 info |= MAY_BE_RCN;
2020 }
2021 if (info & MAY_BE_UNDEF) {
2022 info |= MAY_BE_NULL;
2023 info &= ~MAY_BE_UNDEF;
2024 }
2025 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2026 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2027 }
2028 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2029 }
2030 break;
2031 case ZEND_RETURN:
2032 ADD_OP1_TRACE_GUARD();
2033 /* Propagate return value types */
2034 if (opline->op1_type == IS_UNUSED) {
2035 return_value_info.type = MAY_BE_NULL;
2036 } else if (opline->op1_type == IS_CONST) {
2037 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2038 } else {
2039 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2040 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2041 if (return_value_info.type & MAY_BE_UNDEF) {
2042 return_value_info.type &= ~MAY_BE_UNDEF;
2043 return_value_info.type |= MAY_BE_NULL;
2044 }
2045 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2046 /* CVs are going to be destructed and the reference-counter
2047 of return value may be decremented to 1 */
2048 return_value_info.type |= MAY_BE_RC1;
2049 }
2050 return_value_info.type &= ~MAY_BE_GUARD;
2051 }
2052 break;
2053 case ZEND_CHECK_FUNC_ARG:
2054 if (!frame
2055 || !frame->call
2056 || !frame->call->func) {
2057 break;
2058 }
2059 if (opline->op2_type == IS_CONST
2060 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2061 /* Named parameters not supported in JIT */
2062 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2063 break;
2064 }
2065 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2066 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2067 } else {
2068 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2069 }
2070 break;
2071 case ZEND_FETCH_OBJ_FUNC_ARG:
2072 if (!frame
2073 || !frame->call
2074 || !frame->call->func
2075 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2076 break;
2077 }
2078 ZEND_FALLTHROUGH;
2079 case ZEND_FETCH_OBJ_R:
2080 case ZEND_FETCH_OBJ_IS:
2081 case ZEND_FETCH_OBJ_W:
2082 if (opline->op2_type != IS_CONST
2083 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2084 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2085 break;
2086 }
2087 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2088 ADD_OP1_TRACE_GUARD();
2089 }
2090 break;
2091 case ZEND_INIT_METHOD_CALL:
2092 if (opline->op2_type != IS_CONST
2093 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2094 break;
2095 }
2096 ADD_OP1_TRACE_GUARD();
2097 break;
2098 case ZEND_INIT_DYNAMIC_CALL:
2099 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2100 ADD_OP2_TRACE_GUARD();
2101 }
2102 break;
2103 case ZEND_SEND_ARRAY:
2104 case ZEND_SEND_UNPACK:
2105 case ZEND_CHECK_UNDEF_ARGS:
2106 case ZEND_INCLUDE_OR_EVAL:
2107 max_used_stack = used_stack = -1;
2108 break;
2109 case ZEND_TYPE_CHECK:
2110 if (opline->extended_value == MAY_BE_RESOURCE) {
2111 // TODO: support for is_resource() ???
2112 break;
2113 }
2114 if (op1_type != IS_UNKNOWN
2115 && (opline->extended_value == (1 << op1_type)
2116 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2117 /* add guards only for exact checks, to avoid code duplication */
2118 ADD_OP1_TRACE_GUARD();
2119 }
2120 break;
2121 case ZEND_ROPE_INIT:
2122 case ZEND_ROPE_ADD:
2123 case ZEND_ROPE_END:
2124 if (op2_type == IS_STRING) {
2125 ADD_OP2_TRACE_GUARD();
2126 }
2127 break;
2128 default:
2129 break;
2130 }
2131 len = zend_jit_trace_op_len(opline);
2132 if (ssa->var_info) {
2133 /* Add statically inferred ranges */
2134 if (ssa_ops[idx].op1_def >= 0) {
2135 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2136 }
2137 if (ssa_ops[idx].op2_def >= 0) {
2138 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2139 }
2140 if (ssa_ops[idx].result_def >= 0) {
2141 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2142 }
2143 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2144 if (ssa_ops[idx+1].op1_def >= 0) {
2145 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2146 }
2147 if (ssa_ops[idx+1].op2_def >= 0) {
2148 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2149 }
2150 if (ssa_ops[idx+1].result_def >= 0) {
2151 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2152 }
2153 }
2154 } else {
2155 if (ssa_ops[idx].op1_def >= 0) {
2156 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2157 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2158 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2159 }
2160 }
2161 if (ssa_ops[idx].op2_def >= 0) {
2162 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2163 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2164 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2165 }
2166 }
2167 if (ssa_ops[idx].result_def >= 0) {
2168 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2169 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2170 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2171 }
2172 }
2173 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2174 if (ssa_ops[idx+1].op1_def >= 0) {
2175 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2176 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2177 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2178 }
2179 }
2180 if (ssa_ops[idx+1].op2_def >= 0) {
2181 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2182 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2183 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2184 }
2185 }
2186 if (ssa_ops[idx+1].result_def >= 0) {
2187 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2188 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2189 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2190 }
2191 }
2192 }
2193 }
2194 if (opline->opcode == ZEND_RECV_INIT
2195 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2196 /* RECV_INIT always copy the constant */
2197 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2198 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2199 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2200 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2201 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2202 }
2203 } else {
2204 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2205 // TODO:
2206 assert(0);
2207 }
2208 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2209 && ssa_ops[idx].op1_def >= 0
2210 && op1_type == IS_ARRAY
2211 && (orig_op1_type & IS_TRACE_PACKED)
2212 && val_type != IS_UNKNOWN
2213 && val_type != IS_UNDEF
2214 && ((opline->op2_type == IS_CONST
2215 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2216 || (opline->op2_type != IS_CONST
2217 && op2_type == IS_LONG))) {
2218 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2219
2220 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2221 }
2222 }
2223 if (ssa->var_info) {
2224 /* Add statically inferred restrictions */
2225 if (ssa_ops[idx].op1_def >= 0) {
2226 if (opline->opcode == ZEND_SEND_VAR_EX
2227 && frame
2228 && frame->call
2229 && frame->call->func
2230 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2231 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2232 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2233 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2234 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2235 }
2236 } else {
2237 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2238 }
2239 }
2240 if (ssa_ops[idx].op2_def >= 0) {
2241 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2242 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2243 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2244 }
2245 }
2246 if (ssa_ops[idx].result_def >= 0) {
2247 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2248 }
2249 }
2250 idx++;
2251 while (len > 1) {
2252 opline++;
2253 if (opline->opcode != ZEND_OP_DATA) {
2254 if (ssa->var_info) {
2255 /* Add statically inferred ranges */
2256 if (ssa_ops[idx].op1_def >= 0) {
2257 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2258 }
2259 if (ssa_ops[idx].op2_def >= 0) {
2260 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2261 }
2262 if (ssa_ops[idx].result_def >= 0) {
2263 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2264 }
2265 } else {
2266 if (ssa_ops[idx].op1_def >= 0) {
2267 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2268 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2269 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2270 }
2271 }
2272 if (ssa_ops[idx].op2_def >= 0) {
2273 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2274 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2275 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2276 }
2277 }
2278 if (ssa_ops[idx].result_def >= 0) {
2279 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2280 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2281 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2282 }
2283 }
2284 }
2285 if (opline->opcode == ZEND_RECV_INIT
2286 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2287 /* RECV_INIT always copy the constant */
2288 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2289 } else {
2290 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2291 // TODO:
2292 assert(0);
2293 }
2294 }
2295 }
2296 if (ssa->var_info) {
2297 /* Add statically inferred restrictions */
2298 if (ssa_ops[idx].op1_def >= 0) {
2299 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2300 }
2301 if (ssa_ops[idx].op2_def >= 0) {
2302 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2303 }
2304 if (ssa_ops[idx].result_def >= 0) {
2305 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2306 }
2307 }
2308 idx++;
2309 len--;
2310 }
2311
2312 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2313 op_array = p->op_array;
2314 jit_extension =
2315 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2316 ssa = &jit_extension->func_info.ssa;
2317
2318 call = frame->call;
2319 if (!call) {
2320 /* Trace missed INIT_FCALL opcode */
2321 call = top;
2322 TRACE_FRAME_INIT(call, op_array, 0, 0);
2323 call->used_stack = 0;
2324 top = zend_jit_trace_call_frame(top, op_array);
2325 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2326 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
2327 }
2328 } else {
2329 ZEND_ASSERT(&call->func->op_array == op_array);
2330 }
2331 frame->call = call->prev;
2332 call->prev = frame;
2333 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2334 frame = call;
2335
2336 level++;
2337 i = 0;
2338 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2339 while (i < op_array->last_var) {
2340 ssa_vars[v].var = i;
2341 if (i < op_array->num_args) {
2342 if (ssa->var_info
2343 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2344 /* pass */
2345 } else {
2346 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2347 if (op_array->arg_info) {
2348 zend_arg_info *arg_info = &op_array->arg_info[i];
2349 zend_class_entry *ce;
2350 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2351
2352 if (ZEND_ARG_SEND_MODE(arg_info)) {
2353 tmp |= MAY_BE_REF;
2354 }
2355 ssa_var_info[v].type = tmp;
2356 ssa_var_info[v].ce = ce;
2357 ssa_var_info[v].is_instanceof = 1;
2358 } else {
2359 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2360 }
2361 }
2362 } else {
2363 if (ssa->vars) {
2364 ssa_vars[v].no_val = ssa->vars[i].no_val;
2365 ssa_vars[v].alias = ssa->vars[i].alias;
2366 } else {
2367 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2368 }
2369 if (ssa_vars[v].alias == NO_ALIAS) {
2370 ssa_var_info[v].type = MAY_BE_UNDEF;
2371 } else {
2372 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2373 }
2374 }
2375 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2376 && i < op_array->num_args) {
2377 /* Propagate argument type */
2378 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2379 }
2380 i++;
2381 v++;
2382 }
2383 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2384 op_array = p->op_array;
2385 jit_extension =
2386 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2387 ssa = &jit_extension->func_info.ssa;
2388 if (level == 0) {
2389 i = 0;
2390 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2391 while (i < op_array->last_var) {
2392 ssa_vars[v].var = i;
2393 if (!ssa->var_info
2394 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2395 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2396 }
2397 i++;
2398 v++;
2399 }
2400 while (i < op_array->last_var + op_array->T) {
2401 ssa_vars[v].var = i;
2402 if (!ssa->var_info
2403 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2404 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2405 }
2406 i++;
2407 v++;
2408 }
2409 if (return_value_info.type != 0) {
2410 zend_jit_trace_rec *q = p + 1;
2411 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2412 q++;
2413 }
2414 if (q->op == ZEND_JIT_TRACE_VM
2415 || (q->op == ZEND_JIT_TRACE_END
2416 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2417 const zend_op *opline = q->opline - 1;
2418 if (opline->result_type != IS_UNUSED) {
2419 ssa_var_info[
2420 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2421 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2422 }
2423 }
2424 memset(&return_value_info, 0, sizeof(return_value_info));
2425 }
2426 } else {
2427 level--;
2428 if (return_value_info.type != 0) {
2429 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2430 const zend_op *opline = (p+1)->opline - 1;
2431 if (opline->result_type != IS_UNUSED) {
2432 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2433 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2434 }
2435 }
2436 }
2437 memset(&return_value_info, 0, sizeof(return_value_info));
2438 }
2439 }
2440
2441 top = frame;
2442 if (frame->prev) {
2443 if (used_stack > 0) {
2444 used_stack -= frame->used_stack;
2445 }
2446 frame = frame->prev;
2447 ZEND_ASSERT(&frame->func->op_array == op_array);
2448 } else {
2449 max_used_stack = used_stack = -1;
2450 frame = zend_jit_trace_ret_frame(frame, op_array);
2451 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2452 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2453 frame->used_stack = 0;
2454 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2455 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
2456 }
2457 }
2458
2459 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2460 call = top;
2461 TRACE_FRAME_INIT(call, p->func, 0, 0);
2462 call->prev = frame->call;
2463 call->used_stack = 0;
2464 frame->call = call;
2465 top = zend_jit_trace_call_frame(top, p->op_array);
2466 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2467 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2468 SET_STACK_INFO(call->stack, i, -1);
2469 }
2470 }
2471 if (used_stack >= 0
2472 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2473 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2474 max_used_stack = used_stack = -1;
2475 } else {
2476 const zend_op *opline = (p-1)->opline;
2477
2478 switch (opline->opcode) {
2479 case ZEND_INIT_FCALL:
2480 case ZEND_INIT_FCALL_BY_NAME:
2481 case ZEND_INIT_NS_FCALL_BY_NAME:
2482 case ZEND_INIT_METHOD_CALL:
2483 case ZEND_INIT_DYNAMIC_CALL:
2484 //case ZEND_INIT_STATIC_METHOD_CALL:
2485 //case ZEND_INIT_USER_CALL:
2486 //case ZEND_NEW:
2487 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2488 used_stack += frame->used_stack;
2489 if (used_stack > max_used_stack) {
2490 max_used_stack = used_stack;
2491 }
2492 break;
2493 default:
2494 max_used_stack = used_stack = -1;
2495 }
2496 }
2497 }
2498 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2499 call = frame->call;
2500 if (call) {
2501 top = call;
2502 frame->call = call->prev;
2503 }
2504
2505 if (idx > 0
2506 && ssa_ops[idx-1].result_def >= 0
2507 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2508 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2509 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2510 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2511 opline->opcode == ZEND_DO_FCALL ||
2512 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2513
2514 if (opline->result_type != IS_UNDEF) {
2515 zend_class_entry *ce;
2516 const zend_function *func = p->func;
2517 zend_arg_info *ret_info = func->common.arg_info - 1;
2518 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2519
2520 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2521 }
2522 }
2523 } else if (p->op == ZEND_JIT_TRACE_END) {
2524 break;
2525 }
2526 }
2527
2528 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2529
2530 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2531 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2532 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2533 /* Propagate guards through Phi sources */
2534 zend_ssa_phi *phi = tssa->blocks[1].phis;
2535
2536 op_array = trace_buffer->op_array;
2537 jit_extension =
2538 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2539 ssa = &jit_extension->func_info.ssa;
2540
2541 while (phi) {
2542 uint32_t t = ssa_var_info[phi->ssa_var].type;
2543
2544 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2545 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2546 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2547
2548 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2549 if (!((t0 | t1) & MAY_BE_GUARD)) {
2550 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2551 }
2552 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2553 if (!(t1 & MAY_BE_GUARD)
2554 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2555 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2556 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2557 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2558 MAY_BE_GUARD;
2559 if (!(t0 & MAY_BE_ARRAY)) {
2560 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2561 }
2562 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2563 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2564 }
2565 ssa_var_info[phi->sources[0]].type = t0;
2566 ssa_var_info[phi->sources[0]].type = t0;
2567 }
2568 } else {
2569 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2570 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2571 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2572 MAY_BE_GUARD;
2573 if (!(t0 & MAY_BE_ARRAY)) {
2574 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2575 }
2576 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2577 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2578 }
2579 ssa_var_info[phi->sources[0]].type = t0;
2580 }
2581 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2582 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2583 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2584 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2585 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2586 MAY_BE_GUARD;
2587 if (!(t1 & MAY_BE_ARRAY)) {
2588 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2589 }
2590 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2591 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2592 }
2593 ssa_var_info[phi->sources[1]].type = t1;
2594 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2595 }
2596 }
2597 }
2598 t = ssa_var_info[phi->ssa_var].type;
2599 }
2600
2601 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2602 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2603 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2604
2605 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2606 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2607 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2608 }
2609 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2610 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2611 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2612 ssa_var_info[phi->sources[0]].type =
2613 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2614 }
2615 }
2616 }
2617 phi = phi->next;
2618 }
2619 }
2620
2621 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2622 if (parent_trace) {
2623 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2624 ZEND_JIT_TRACE_NUM,
2625 parent_trace,
2626 exit_num,
2627 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2628 trace_buffer->op_array->scope ? "::" : "",
2629 trace_buffer->op_array->function_name ?
2630 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2631 ZSTR_VAL(trace_buffer->op_array->filename),
2632 trace_buffer[1].opline->lineno);
2633 } else {
2634 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2635 ZEND_JIT_TRACE_NUM,
2636 zend_jit_trace_star_desc(trace_buffer->start),
2637 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2638 trace_buffer->op_array->scope ? "::" : "",
2639 trace_buffer->op_array->function_name ?
2640 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2641 ZSTR_VAL(trace_buffer->op_array->filename),
2642 trace_buffer[1].opline->lineno);
2643 }
2644 zend_jit_dump_trace(trace_buffer, tssa);
2645 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2646 uint32_t idx = trace_buffer[1].last;
2647 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2648 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2649 ZEND_JIT_TRACE_NUM,
2650 link_to);
2651 } else {
2652 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2653 ZEND_JIT_TRACE_NUM,
2654 zend_jit_trace_stop_description[trace_buffer->stop]);
2655 }
2656 }
2657
2658 return tssa;
2659 }
2660
zend_jit_close_var(zend_jit_trace_stack * stack,uint32_t n,int * start,int * end,uint8_t * flags,int line)2661 static void zend_jit_close_var(zend_jit_trace_stack *stack, uint32_t n, int *start, int *end, uint8_t *flags, int line)
2662 {
2663 int32_t var = STACK_VAR(stack, n);
2664
2665 if (var >= 0 && start[var] >= 0 && !(flags[var] & ZREG_LAST_USE)) {
2666 // TODO: shrink interval to last side exit ????
2667 end[var] = line;
2668 }
2669 }
2670
zend_jit_trace_use_var(int line,int var,int def,int use_chain,int * start,int * end,uint8_t * flags,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2671 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, int *start, int *end, uint8_t *flags, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2672 {
2673 ZEND_ASSERT(start[var] >= 0);
2674 ZEND_ASSERT(!(flags[var] & ZREG_LAST_USE));
2675 end[var] = line;
2676 if (def >= 0) {
2677 flags[var] |= ZREG_LAST_USE;
2678 } else if (use_chain < 0 && (flags[var] & (ZREG_LOAD|ZREG_STORE))) {
2679 flags[var] |= ZREG_LAST_USE;
2680 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2681 /* pass */
2682 } else if (op_array_ssa->vars) {
2683 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2684
2685 if (ssa->ops[line].op1_use == var) {
2686 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2687 flags[var] |= ZREG_LAST_USE;
2688 }
2689 } else if (ssa->ops[line].op2_use == var) {
2690 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2691 flags[var] |= ZREG_LAST_USE;
2692 }
2693 } else if (ssa->ops[line].result_use == var) {
2694 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2695 flags[var] |= ZREG_LAST_USE;
2696 }
2697 }
2698 }
2699 }
2700
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2701 static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2702 {
2703 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2704 zend_jit_trace_rec *p;
2705 const zend_op_array *op_array;
2706 zend_jit_op_array_trace_extension *jit_extension;
2707 const zend_ssa *op_array_ssa;
2708 const zend_ssa_op *ssa_op;
2709 int i, j, idx, count, level;
2710 int last_idx = -1;
2711 int *start, *end;
2712 uint8_t *flags;
2713 const zend_op_array **vars_op_array;
2714 zend_lifetime_interval **intervals, *list, *ival;
2715 void *checkpoint;
2716 zend_jit_trace_stack_frame *frame;
2717 zend_jit_trace_stack *stack;
2718 uint32_t parent_vars_count = parent_trace ?
2719 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2720 zend_jit_trace_stack *parent_stack = parent_trace ?
2721 zend_jit_traces[parent_trace].stack_map +
2722 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2723 ALLOCA_FLAG(use_heap);
2724
2725 ZEND_ASSERT(ssa->var_info != NULL);
2726
2727 start = do_alloca(sizeof(int) * ssa->vars_count * 2 +
2728 ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count),
2729 use_heap);
2730 if (!start) {
2731 return NULL;
2732 }
2733 end = start + ssa->vars_count;
2734 flags = (uint8_t*)(end + ssa->vars_count);
2735 checkpoint = zend_arena_checkpoint(CG(arena));
2736 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2737
2738 memset(start, -1, sizeof(int) * ssa->vars_count * 2);
2739 memset(flags, 0, sizeof(uint8_t) * ssa->vars_count);
2740 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2741
2742 op_array = trace_buffer->op_array;
2743 jit_extension =
2744 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2745 op_array_ssa = &jit_extension->func_info.ssa;
2746 frame = JIT_G(current_frame);
2747 frame->prev = NULL;
2748 frame->func = (const zend_function*)op_array;
2749 stack = frame->stack;
2750
2751 count = 0;
2752
2753 i = 0;
2754 j = op_array->last_var;
2755 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2756 j += op_array->T;
2757 }
2758 while (i < j) {
2759 SET_STACK_VAR(stack, i, i);
2760 vars_op_array[i] = op_array;
2761 /* We don't start intervals for variables used in Phi */
2762 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2763 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2764 && ssa->vars[i].alias == NO_ALIAS
2765 && zend_jit_var_supports_reg(ssa, i)) {
2766 start[i] = 0;
2767 if (i < parent_vars_count
2768 && STACK_REG(parent_stack, i) != ZREG_NONE
2769 && STACK_REG(parent_stack, i) < ZREG_NUM) {
2770 /* We will try to reuse register from parent trace */
2771 flags[i] = STACK_FLAGS(parent_stack, i);
2772 count += 2;
2773 } else {
2774 flags[i] = ZREG_LOAD;
2775 count++;
2776 }
2777 }
2778 i++;
2779 }
2780
2781 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2782 j = op_array->last_var + op_array->T;
2783 while (i < j) {
2784 SET_STACK_VAR(stack, i, -1);
2785 i++;
2786 }
2787 }
2788
2789 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2790 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2791 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2792 zend_ssa_phi *phi = ssa->blocks[1].phis;
2793
2794 while (phi) {
2795 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2796 vars_op_array[phi->ssa_var] = op_array;
2797 if (ssa->vars[phi->ssa_var].use_chain >= 0
2798 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2799 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2800 start[phi->ssa_var] = 0;
2801 count++;
2802 }
2803 phi = phi->next;
2804 }
2805 }
2806
2807 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2808 level = 0;
2809 ssa_op = ssa->ops;
2810 idx = 0;
2811 for (;;p++) {
2812 if (p->op == ZEND_JIT_TRACE_VM) {
2813 const zend_op *opline = p->opline;
2814 int len;
2815 bool support_opline;
2816
2817 support_opline =
2818 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2819
2820 if (support_opline
2821 && opline->opcode == ZEND_ASSIGN
2822 && opline->op1_type == IS_CV
2823 && ssa_op->op1_def >= 0
2824 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2825 /* avoid register allocation in case of possibility of indirect modification*/
2826 support_opline = 0;
2827 }
2828
2829 if (ssa_op->op1_use >= 0
2830 && start[ssa_op->op1_use] >= 0
2831 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2832 if (support_opline) {
2833 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2834 if (opline->op1_type != IS_CV) {
2835 if (opline->opcode == ZEND_CASE
2836 || opline->opcode == ZEND_CASE_STRICT
2837 || opline->opcode == ZEND_SWITCH_LONG
2838 || opline->opcode == ZEND_MATCH
2839 || opline->opcode == ZEND_FETCH_LIST_R
2840 || opline->opcode == ZEND_COPY_TMP
2841 || opline->opcode == ZEND_SWITCH_STRING
2842 || opline->opcode == ZEND_FE_FETCH_R
2843 || opline->opcode == ZEND_FE_FETCH_RW
2844 || opline->opcode == ZEND_FETCH_LIST_W
2845 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2846 || opline->opcode == ZEND_BIND_LEXICAL
2847 || opline->opcode == ZEND_ROPE_ADD) {
2848 /* The value is kept alive and may be used outside of the trace */
2849 flags[ssa_op->op1_use] |= ZREG_STORE;
2850 } else {
2851 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2852 }
2853 }
2854 } else {
2855 start[ssa_op->op1_use] = -1;
2856 end[ssa_op->op1_use] = -1;
2857 count--;
2858 }
2859 }
2860 if (ssa_op->op2_use >= 0
2861 && ssa_op->op2_use != ssa_op->op1_use
2862 && start[ssa_op->op2_use] >= 0
2863 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2864 if (support_opline) {
2865 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2866 if (opline->op2_type != IS_CV) {
2867 flags[ssa_op->op2_use] |= ZREG_LAST_USE;
2868 }
2869 } else {
2870 start[ssa_op->op2_use] = -1;
2871 end[ssa_op->op2_use] = -1;
2872 count--;
2873 }
2874 }
2875 if (ssa_op->result_use >= 0
2876 && ssa_op->result_use != ssa_op->op1_use
2877 && ssa_op->result_use != ssa_op->op2_use
2878 && start[ssa_op->result_use] >= 0
2879 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2880 if (support_opline) {
2881 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2882 } else {
2883 start[ssa_op->result_use] = -1;
2884 end[ssa_op->result_use] = -1;
2885 count--;
2886 }
2887 }
2888
2889 if (ssa_op->op1_def >= 0) {
2890 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2891 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2892 }
2893 if (ssa_op->op2_def >= 0) {
2894 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op2.var), start, end, flags, idx);
2895 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
2896 }
2897 if (ssa_op->result_def >= 0) {
2898 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2899 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2900 }
2901
2902 if (support_opline) {
2903 if (ssa_op->result_def >= 0
2904 && (ssa->vars[ssa_op->result_def].use_chain >= 0
2905 || ssa->vars[ssa_op->result_def].phi_use_chain)
2906 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
2907 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
2908 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
2909 || opline->opcode == ZEND_PRE_INC
2910 || opline->opcode == ZEND_PRE_DEC
2911 || opline->opcode == ZEND_POST_INC
2912 || opline->opcode == ZEND_POST_DEC
2913 || opline->opcode == ZEND_ADD
2914 || opline->opcode == ZEND_SUB
2915 || opline->opcode == ZEND_MUL
2916 || opline->opcode == ZEND_FETCH_DIM_R
2917 || opline->opcode == ZEND_FETCH_CONSTANT) {
2918 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
2919 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
2920 start[ssa_op->result_def] = idx;
2921 vars_op_array[ssa_op->result_def] = op_array;
2922 count++;
2923 }
2924 }
2925 }
2926 if (ssa_op->op1_def >= 0
2927 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2928 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2929 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2930 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2931 start[ssa_op->op1_def] = idx;
2932 vars_op_array[ssa_op->op1_def] = op_array;
2933 count++;
2934 }
2935 if (ssa_op->op2_def >= 0
2936 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
2937 || ssa->vars[ssa_op->op2_def].phi_use_chain)
2938 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
2939 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)) {
2940 start[ssa_op->op2_def] = idx;
2941 vars_op_array[ssa_op->op2_def] = op_array;
2942 count++;
2943 }
2944 }
2945
2946 len = zend_jit_trace_op_len(opline);
2947 switch (opline->opcode) {
2948 case ZEND_ASSIGN_DIM:
2949 case ZEND_ASSIGN_OBJ:
2950 case ZEND_ASSIGN_STATIC_PROP:
2951 case ZEND_ASSIGN_DIM_OP:
2952 case ZEND_ASSIGN_OBJ_OP:
2953 case ZEND_ASSIGN_STATIC_PROP_OP:
2954 case ZEND_ASSIGN_OBJ_REF:
2955 case ZEND_ASSIGN_STATIC_PROP_REF:
2956 /* OP_DATA */
2957 ssa_op++;
2958 opline++;
2959 if (ssa_op->op1_use >= 0
2960 && start[ssa_op->op1_use] >= 0
2961 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2962 if (support_opline) {
2963 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2964 if (opline->op1_type != IS_CV) {
2965 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2966 }
2967 } else {
2968 start[ssa_op->op1_use] = -1;
2969 end[ssa_op->op1_use] = -1;
2970 count--;
2971 }
2972 }
2973 if (ssa_op->op1_def >= 0) {
2974 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2975 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2976 if (support_opline
2977 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2978 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2979 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2980 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2981 start[ssa_op->op1_def] = idx;
2982 vars_op_array[ssa_op->op1_def] = op_array;
2983 count++;
2984 }
2985 }
2986 ssa_op++;
2987 opline++;
2988 idx+=2;
2989 break;
2990 case ZEND_RECV_INIT:
2991 ssa_op++;
2992 opline++;
2993 idx++;
2994 while (opline->opcode == ZEND_RECV_INIT) {
2995 /* RECV_INIT doesn't support registers */
2996 if (ssa_op->result_def >= 0) {
2997 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2998 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2999 }
3000 ssa_op++;
3001 opline++;
3002 idx++;
3003 }
3004 break;
3005 case ZEND_BIND_GLOBAL:
3006 ssa_op++;
3007 opline++;
3008 idx++;
3009 while (opline->opcode == ZEND_BIND_GLOBAL) {
3010 /* BIND_GLOBAL doesn't support registers */
3011 if (ssa_op->op1_def >= 0) {
3012 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
3013 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3014 }
3015 ssa_op++;
3016 opline++;
3017 idx++;
3018 }
3019 break;
3020 default:
3021 ssa_op += len;
3022 idx += len;
3023 break;
3024 }
3025 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3026 /* New call frames */
3027 zend_jit_trace_stack_frame *prev_frame = frame;
3028
3029 frame = zend_jit_trace_call_frame(frame, op_array);
3030 frame->prev = prev_frame;
3031 frame->func = (const zend_function*)p->op_array;
3032 stack = frame->stack;
3033 op_array = p->op_array;
3034 jit_extension =
3035 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3036 op_array_ssa = &jit_extension->func_info.ssa;
3037 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3038 for (i = 0; i < op_array->last_var; i++) {
3039 SET_STACK_VAR(stack, i, j);
3040 vars_op_array[j] = op_array;
3041 if (ssa->vars[j].use_chain >= 0
3042 && ssa->vars[j].alias == NO_ALIAS
3043 && zend_jit_var_supports_reg(ssa, j)) {
3044 start[j] = idx;
3045 flags[j] = ZREG_LOAD;
3046 count++;
3047 }
3048 j++;
3049 }
3050 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3051 SET_STACK_VAR(stack, i, -1);
3052 }
3053 level++;
3054 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3055 /* Close exiting call frames */
3056 for (i = 0; i < op_array->last_var; i++) {
3057 zend_jit_close_var(stack, i, start, end, flags, idx-1);
3058 }
3059 op_array = p->op_array;
3060 jit_extension =
3061 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3062 op_array_ssa = &jit_extension->func_info.ssa;
3063 frame = zend_jit_trace_ret_frame(frame, op_array);
3064 stack = frame->stack;
3065 if (level == 0) {
3066 /* New return frames */
3067 frame->prev = NULL;
3068 frame->func = (const zend_function*)op_array;
3069 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3070 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3071 SET_STACK_VAR(stack, i, j);
3072 vars_op_array[j] = op_array;
3073 if (ssa->vars[j].use_chain >= 0
3074 && ssa->vars[j].alias == NO_ALIAS
3075 && zend_jit_var_supports_reg(ssa, j)) {
3076 start[j] = idx;
3077 flags[j] = ZREG_LOAD;
3078 count++;
3079 }
3080 j++;
3081 }
3082 } else {
3083 level--;
3084 }
3085 } else if (p->op == ZEND_JIT_TRACE_END) {
3086 break;
3087 }
3088 }
3089
3090 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3091 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3092 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3093 zend_ssa_phi *phi = ssa->blocks[1].phis;
3094
3095 while (phi) {
3096 i = phi->sources[1];
3097 if (start[i] >= 0 && !ssa->vars[phi->ssa_var].no_val) {
3098 end[i] = idx;
3099 flags[i] &= ~ZREG_LAST_USE;
3100 }
3101 phi = phi->next;
3102 }
3103
3104 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3105 for (i = 0; i < op_array->last_var; i++) {
3106 if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) {
3107 end[i] = idx;
3108 flags[i] &= ~ZREG_LAST_USE;
3109 } else {
3110 zend_jit_close_var(stack, i, start, end, flags, idx);
3111 }
3112 }
3113 }
3114 } else {
3115 last_idx = idx;
3116 for (i = 0; i < op_array->last_var; i++) {
3117 zend_jit_close_var(stack, i, start, end, flags, idx);
3118 }
3119 while (frame->prev) {
3120 frame = frame->prev;
3121 op_array = &frame->func->op_array;
3122 stack = frame->stack;
3123 for (i = 0; i < op_array->last_var; i++) {
3124 zend_jit_close_var(stack, i, start, end, flags, idx);
3125 }
3126 }
3127 }
3128
3129 if (!count) {
3130 free_alloca(start, use_heap);
3131 zend_arena_release(&CG(arena), checkpoint);
3132 return NULL;
3133 }
3134
3135 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval));
3136 memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count);
3137 list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count);
3138 j = 0;
3139 for (i = 0; i < ssa->vars_count; i++) {
3140 if (start[i] >= 0 && end[i] >= 0) {
3141 ZEND_ASSERT(j < count);
3142 if ((flags[i] & ZREG_LOAD) &&
3143 (flags[i] & ZREG_LAST_USE) &&
3144 end[i] == ssa->vars[i].use_chain) {
3145 /* skip life range with single use */
3146 continue;
3147 }
3148 intervals[i] = &list[j];
3149 list[j].ssa_var = i;
3150 list[j].reg = ZREG_NONE;
3151 list[j].flags = flags[i];
3152 list[j].range.start = start[i];
3153 list[j].range.end = end[i];
3154 list[j].range.next = NULL;
3155 list[j].hint = NULL;
3156 list[j].used_as_hint = NULL;
3157 list[j].list_next = NULL;
3158 j++;
3159 }
3160 }
3161 count = j;
3162 free_alloca(start, use_heap);
3163 start = end = NULL;
3164
3165 if (!count) {
3166 zend_arena_release(&CG(arena), checkpoint);
3167 return NULL;
3168 }
3169
3170 /* Add hints */
3171 if (parent_vars_count) {
3172 i = trace_buffer->op_array->last_var;
3173 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3174 i += trace_buffer->op_array->T;
3175 }
3176 if ((uint32_t)i > parent_vars_count) {
3177 i = parent_vars_count;
3178 }
3179 while (i > 0) {
3180 i--;
3181 if (intervals[i]
3182 && STACK_REG(parent_stack, i) != ZREG_NONE
3183 && STACK_REG(parent_stack, i) < ZREG_NUM) {
3184 list[j].ssa_var = - 1;
3185 list[j].reg = STACK_REG(parent_stack, i);
3186 list[j].flags = 0;
3187 list[j].range.start = -1;
3188 list[j].range.end = -1;
3189 list[j].range.next = NULL;
3190 list[j].hint = NULL;
3191 list[j].used_as_hint = NULL;
3192 list[j].list_next = NULL;
3193 intervals[i]->hint = &list[j];
3194 j++;
3195 }
3196 }
3197 }
3198
3199 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3200 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3201 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3202 zend_ssa_phi *phi = ssa->blocks[1].phis;
3203
3204 while (phi) {
3205 if (intervals[phi->ssa_var]) {
3206 if (intervals[phi->sources[1]]
3207 && (ssa->var_info[phi->sources[1]].type & MAY_BE_ANY) ==
3208 (ssa->var_info[phi->ssa_var].type & MAY_BE_ANY)) {
3209 intervals[phi->sources[1]]->hint = intervals[phi->ssa_var];
3210 }
3211 }
3212 phi = phi->next;
3213 }
3214 }
3215
3216 for (i = 0; i < ssa->vars_count; i++) {
3217 if (intervals[i] && !intervals[i]->hint) {
3218
3219 if (ssa->vars[i].definition >= 0) {
3220 uint32_t line = ssa->vars[i].definition;
3221 const zend_op *opline = ssa_opcodes[line];
3222
3223 switch (opline->opcode) {
3224 case ZEND_QM_ASSIGN:
3225 case ZEND_POST_INC:
3226 case ZEND_POST_DEC:
3227 if (ssa->ops[line].op1_use >= 0 &&
3228 intervals[ssa->ops[line].op1_use] &&
3229 (i == ssa->ops[line].op1_def ||
3230 (i == ssa->ops[line].result_def &&
3231 (ssa->ops[line].op1_def < 0 ||
3232 !intervals[ssa->ops[line].op1_def])))) {
3233 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3234 }
3235 break;
3236 case ZEND_SEND_VAR:
3237 if (opline->op2_type == IS_CONST) {
3238 /* Named parameters not supported in JIT */
3239 break;
3240 }
3241 case ZEND_PRE_INC:
3242 case ZEND_PRE_DEC:
3243 if (i == ssa->ops[line].op1_def &&
3244 ssa->ops[line].op1_use >= 0 &&
3245 intervals[ssa->ops[line].op1_use]) {
3246 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3247 }
3248 break;
3249 case ZEND_ASSIGN:
3250 if (ssa->ops[line].op2_use >= 0 &&
3251 intervals[ssa->ops[line].op2_use] &&
3252 (i == ssa->ops[line].op2_def ||
3253 (i == ssa->ops[line].op1_def &&
3254 (ssa->ops[line].op2_def < 0 ||
3255 !intervals[ssa->ops[line].op2_def])) ||
3256 (i == ssa->ops[line].result_def &&
3257 (ssa->ops[line].op2_def < 0 ||
3258 !intervals[ssa->ops[line].op2_def]) &&
3259 (ssa->ops[line].op1_def < 0 ||
3260 !intervals[ssa->ops[line].op1_def])))) {
3261 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3262 }
3263 break;
3264 case ZEND_SUB:
3265 case ZEND_ADD:
3266 case ZEND_MUL:
3267 case ZEND_BW_OR:
3268 case ZEND_BW_AND:
3269 case ZEND_BW_XOR:
3270 if (i == ssa->ops[line].result_def) {
3271 if (ssa->ops[line].op1_use >= 0 &&
3272 intervals[ssa->ops[line].op1_use] &&
3273 ssa->ops[line].op1_use_chain < 0 &&
3274 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
3275 (ssa->var_info[i].type & MAY_BE_ANY) ==
3276 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
3277
3278 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op1_use].definition_phi;
3279 if (phi &&
3280 intervals[phi->sources[1]] &&
3281 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op1_use]) {
3282 break;
3283 }
3284 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3285 } else if (opline->opcode != ZEND_SUB &&
3286 ssa->ops[line].op2_use >= 0 &&
3287 intervals[ssa->ops[line].op2_use] &&
3288 ssa->ops[line].op2_use_chain < 0 &&
3289 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
3290 (ssa->var_info[i].type & MAY_BE_ANY) ==
3291 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
3292
3293 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op2_use].definition_phi;
3294 if (phi &&
3295 intervals[phi->sources[1]] &&
3296 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op2_use]) {
3297 break;
3298 }
3299 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3300 }
3301 }
3302 break;
3303 }
3304 }
3305 }
3306 }
3307
3308 list = zend_jit_sort_intervals(intervals, ssa->vars_count);
3309
3310 if (list) {
3311 ival = list;
3312 while (ival) {
3313 if (ival->hint) {
3314 ival->hint->used_as_hint = ival;
3315 }
3316 ival = ival->list_next;
3317 }
3318 }
3319
3320 if (list) {
3321 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3322 fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM);
3323 ival = list;
3324 while (ival) {
3325 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3326 ival = ival->list_next;
3327 }
3328 }
3329 }
3330
3331 /* Linear Scan Register Allocation (op_array is not actually used, only fn_flags matters) */
3332 list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list);
3333
3334 if (list) {
3335 zend_lifetime_interval *ival, *next;
3336
3337 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
3338 ival = list;
3339 count = 0;
3340 while (ival != NULL) {
3341 ZEND_ASSERT(ival->reg != ZREG_NONE);
3342 count++;
3343 next = ival->list_next;
3344 ival->list_next = intervals[ival->ssa_var];
3345 intervals[ival->ssa_var] = ival;
3346 ival = next;
3347 }
3348
3349 if (!count) {
3350 zend_arena_release(&CG(arena), checkpoint);
3351 return NULL;
3352 }
3353
3354 /* Add LOAD flag to registers that can't reuse register from parent trace */
3355 if (parent_vars_count) {
3356 i = trace_buffer->op_array->last_var;
3357 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3358 i += trace_buffer->op_array->T;
3359 }
3360 if ((uint32_t)i > parent_vars_count) {
3361 i = parent_vars_count;
3362 }
3363 while (i > 0) {
3364 i--;
3365 if (intervals[i] && intervals[i]->reg != STACK_REG(parent_stack, i)) {
3366 intervals[i]->flags |= ZREG_LOAD;
3367 }
3368 }
3369 }
3370
3371 /* SSA resolution */
3372 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3373 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3374 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3375 zend_ssa_phi *phi = ssa->blocks[1].phis;
3376
3377 while (phi) {
3378 int def = phi->ssa_var;
3379 int use = phi->sources[1];
3380
3381 if (intervals[def]) {
3382 if (!intervals[use]) {
3383 intervals[def]->flags |= ZREG_LOAD;
3384 if ((intervals[def]->flags & ZREG_LAST_USE)
3385 && ssa->vars[def].use_chain >= 0
3386 && ssa->vars[def].use_chain == intervals[def]->range.end) {
3387 /* remove interval used once */
3388 intervals[def] = NULL;
3389 count--;
3390 }
3391 } else if (intervals[def]->reg != intervals[use]->reg) {
3392 intervals[def]->flags |= ZREG_LOAD;
3393 if (ssa->vars[use].use_chain >= 0) {
3394 intervals[use]->flags |= ZREG_STORE;
3395 } else {
3396 intervals[use] = NULL;
3397 count--;
3398 }
3399 } else {
3400 use = phi->sources[0];
3401 ZEND_ASSERT(!intervals[use]);
3402 intervals[use] = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
3403 intervals[use]->ssa_var = phi->sources[0];
3404 intervals[use]->reg = intervals[def]->reg;
3405 intervals[use]->flags = ZREG_LOAD;
3406 intervals[use]->range.start = 0;
3407 intervals[use]->range.end = 0;
3408 intervals[use]->range.next = NULL;
3409 intervals[use]->hint = NULL;
3410 intervals[use]->used_as_hint = NULL;
3411 intervals[use]->list_next = NULL;
3412 }
3413 } else if (intervals[use]
3414 && (!ssa->vars[def].no_val
3415 || ssa->var_info[def].type != ssa->var_info[use].type)) {
3416 if (ssa->vars[use].use_chain >= 0) {
3417 intervals[use]->flags |= ZREG_STORE;
3418 } else {
3419 intervals[use] = NULL;
3420 count--;
3421 }
3422 }
3423 phi = phi->next;
3424 }
3425 } else {
3426 for (i = 0; i < ssa->vars_count; i++) {
3427 if (intervals[i]
3428 && intervals[i]->range.end == last_idx
3429 && !(intervals[i]->flags & (ZREG_LOAD|ZREG_STORE))) {
3430 intervals[i]->flags |= ZREG_STORE;
3431 }
3432 }
3433 }
3434
3435 if (!count) {
3436 zend_arena_release(&CG(arena), checkpoint);
3437 return NULL;
3438 }
3439
3440 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3441 fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM);
3442 for (i = 0; i < ssa->vars_count; i++) {
3443 ival = intervals[i];
3444 while (ival) {
3445 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3446 ival = ival->list_next;
3447 }
3448 }
3449 }
3450
3451 return intervals;
3452 }
3453
3454 zend_arena_release(&CG(arena), checkpoint); //???
3455 return NULL;
3456 }
3457
zend_jit_trace_clenup_stack(zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,zend_lifetime_interval ** ra)3458 static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, zend_lifetime_interval **ra)
3459 {
3460 uint32_t line = ssa_op - ssa->ops;
3461
3462 if (ssa_op->op1_use >= 0
3463 && ra[ssa_op->op1_use]
3464 && ra[ssa_op->op1_use]->range.end == line) {
3465 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
3466 }
3467 if (ssa_op->op2_use >= 0
3468 && ra[ssa_op->op2_use]
3469 && ra[ssa_op->op2_use]->range.end == line) {
3470 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op2.var), ZREG_NONE);
3471 }
3472 if (ssa_op->result_use >= 0
3473 && ra[ssa_op->result_use]
3474 && ra[ssa_op->result_use]->range.end == line) {
3475 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE);
3476 }
3477 }
3478
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3479 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3480 {
3481 zend_op *next_opline = (zend_op*)(opline + 1);
3482
3483 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3484 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3485 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3486 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3487 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3488 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3489 }
3490 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3491 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3492 }
3493 }
3494
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3495 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3496 {
3497 int var = ssa_op->result_def;
3498 int i;
3499 int use = ssa->vars[var].use_chain;
3500 const zend_op *opline;
3501
3502 if (use < 0
3503 || ssa->vars[var].phi_use_chain
3504 || ssa->ops[use].op1_use != var
3505 || ssa->ops[use].op1_use_chain != -1) {
3506 return 0;
3507 }
3508
3509 opline = ssa_opcodes[use];
3510 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3511 return (opline->op2_type == IS_CONST &&
3512 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3513 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3514 if (!JIT_G(current_frame)
3515 || !JIT_G(current_frame)->call
3516 || !JIT_G(current_frame)->call->func
3517 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3518 return 0;
3519 }
3520 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3521 && opline->opcode != ZEND_FETCH_OBJ_IS
3522 && opline->opcode != ZEND_FETCH_OBJ_W
3523 && opline->opcode != ZEND_ASSIGN_OBJ
3524 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3525 && opline->opcode != ZEND_PRE_INC_OBJ
3526 && opline->opcode != ZEND_PRE_DEC_OBJ
3527 && opline->opcode != ZEND_POST_INC_OBJ
3528 && opline->opcode != ZEND_POST_DEC_OBJ) {
3529 return 0;
3530 }
3531
3532 if (opline->op2_type != IS_CONST
3533 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3534 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3535 return 0;
3536 }
3537
3538 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3539 if (opline->op1_type == IS_CV
3540 && (opline+1)->op1_type == IS_CV
3541 && (opline+1)->op1.var == opline->op1.var) {
3542 /* skip $a->prop += $a; */
3543 return 0;
3544 }
3545 if (!zend_jit_supported_binary_op(
3546 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3547 return 0;
3548 }
3549 }
3550
3551 for (i = ssa->vars[var].definition; i < use; i++) {
3552 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3553 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3554 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3555 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3556 return 0;
3557 }
3558 }
3559
3560 return 1;
3561 }
3562
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3563 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3564 {
3565 uint32_t i;
3566
3567 for (i = 0; i < stack_size; i++) {
3568 if (STACK_REG(stack, i) != ZREG_NONE
3569 && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
3570 return 1;
3571 }
3572 }
3573 return 0;
3574 }
3575
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3576 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3577 {
3578 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3579 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3580 uint32_t stack_size;
3581 zend_jit_trace_stack *stack;
3582
3583 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3584 return 1;
3585 }
3586
3587 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3588 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3589 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3590 }
3591
zend_jit_trace_deoptimization(dasm_State ** Dst,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_lifetime_interval ** ra,bool polymorphic_side_trace)3592 static int zend_jit_trace_deoptimization(dasm_State **Dst,
3593 uint32_t flags,
3594 const zend_op *opline,
3595 zend_jit_trace_stack *parent_stack,
3596 int parent_vars_count,
3597 zend_ssa *ssa,
3598 zend_jit_trace_stack *stack,
3599 zend_lifetime_interval **ra,
3600 bool polymorphic_side_trace)
3601 {
3602 int i;
3603 bool has_constants = 0;
3604 bool has_unsaved_vars = 0;
3605
3606 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3607 for (i = 0; i < parent_vars_count; i++) {
3608 int8_t reg = STACK_REG(parent_stack, i);
3609
3610 if (reg != ZREG_NONE) {
3611 if (reg < ZREG_NUM) {
3612 if (ssa && ssa->vars[i].no_val) {
3613 /* pass */
3614 } else if (ra && ra[i] && ra[i]->reg == reg) {
3615 /* register already loaded by parent trace */
3616 if (stack) {
3617 SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i));
3618 }
3619 has_unsaved_vars = 1;
3620 } else {
3621 uint8_t type = STACK_TYPE(parent_stack, i);
3622
3623 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3624 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3625 STACK_MEM_TYPE(parent_stack, i) != type)) {
3626 return 0;
3627 }
3628 if (stack) {
3629 SET_STACK_TYPE(stack, i, type, 1);
3630 }
3631 }
3632 } else {
3633 /* delay custom deoptimization instructions to prevent register clobbering */
3634 has_constants = 1;
3635 }
3636 }
3637 }
3638
3639 if (has_unsaved_vars
3640 && (has_constants
3641 || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) {
3642 for (i = 0; i < parent_vars_count; i++) {
3643 int8_t reg = STACK_REG(parent_stack, i);
3644
3645 if (reg != ZREG_NONE) {
3646 if (reg < ZREG_NUM) {
3647 if (ssa && ssa->vars[i].no_val) {
3648 /* pass */
3649 } else if (ra && ra[i] && ra[i]->reg == reg) {
3650 uint8_t type = STACK_TYPE(parent_stack, i);
3651
3652 if (stack) {
3653 SET_STACK_TYPE(stack, i, type, 1);
3654 }
3655 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3656 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3657 STACK_MEM_TYPE(parent_stack, i) != type)) {
3658 return 0;
3659 }
3660 }
3661 }
3662 }
3663 }
3664 }
3665
3666 if (has_constants) {
3667 for (i = 0; i < parent_vars_count; i++) {
3668 int8_t reg = STACK_REG(parent_stack, i);
3669
3670 if (reg != ZREG_NONE) {
3671 if (reg < ZREG_NUM) {
3672 /* pass */
3673 } else if (reg == ZREG_THIS) {
3674 if (polymorphic_side_trace) {
3675 ssa->var_info[i].delayed_fetch_this = 1;
3676 if (stack) {
3677 SET_STACK_REG(stack, i, ZREG_THIS);
3678 }
3679 } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
3680 return 0;
3681 }
3682 } else {
3683 if (reg == ZREG_ZVAL_COPY_GPR0
3684 &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) {
3685 return 0;
3686 }
3687 if (!zend_jit_store_const(Dst, i, reg)) {
3688 return 0;
3689 }
3690 }
3691 }
3692 }
3693 }
3694
3695 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3696 if (!zend_jit_save_call_chain(Dst, -1)) {
3697 return 0;
3698 }
3699 }
3700
3701 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3702 const zend_op *op = opline - 1;
3703
3704 if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) {
3705 return 0;
3706 }
3707 }
3708
3709 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3710 const zend_op *op = opline - 1;
3711
3712 if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) {
3713 return 0;
3714 }
3715 }
3716
3717 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3718 if (!zend_jit_check_exception(Dst)) {
3719 return 0;
3720 }
3721 }
3722
3723 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3724 if (!zend_jit_free_trampoline(Dst)) {
3725 return 0;
3726 }
3727 }
3728
3729 return 1;
3730 }
3731
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3732 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3733 {
3734 info->has_range = 1;
3735 info->range.min = min;
3736 info->range.max = max;
3737 info->range.underflow = 0;
3738 info->range.overflow = 0;
3739 }
3740
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3741 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3742 {
3743 zend_long op1_min, op1_max, op2_min, op2_max;
3744
3745 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3746 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3747 return;
3748 }
3749
3750 op1_min = OP1_MIN_RANGE();
3751 op1_max = OP1_MAX_RANGE();
3752 op2_min = OP2_MIN_RANGE();
3753 op2_max = OP2_MAX_RANGE();
3754
3755 switch (opline->opcode) {
3756 case ZEND_IS_EQUAL:
3757 case ZEND_CASE:
3758 case ZEND_IS_IDENTICAL:
3759 case ZEND_CASE_STRICT:
3760 case ZEND_IS_NOT_IDENTICAL:
3761 if (!exit_if_true) {
3762 /* op1 == op2 */
3763 if (ssa_op->op1_use >= 0) {
3764 zend_jit_trace_set_var_range(
3765 &ssa->var_info[ssa_op->op1_use],
3766 MAX(op1_min, op2_min),
3767 MIN(op1_max, op2_max));
3768 }
3769 if (ssa_op->op2_use >= 0) {
3770 zend_jit_trace_set_var_range(
3771 &ssa->var_info[ssa_op->op2_use],
3772 MAX(op2_min, op1_min),
3773 MIN(op2_max, op1_max));
3774 }
3775 }
3776 break;
3777 case ZEND_IS_NOT_EQUAL:
3778 if (exit_if_true) {
3779 /* op1 == op2 */
3780 if (ssa_op->op1_use >= 0) {
3781 zend_jit_trace_set_var_range(
3782 &ssa->var_info[ssa_op->op1_use],
3783 MAX(op1_min, op2_min),
3784 MIN(op1_max, op2_max));
3785 }
3786 if (ssa_op->op2_use >= 0) {
3787 zend_jit_trace_set_var_range(
3788 &ssa->var_info[ssa_op->op2_use],
3789 MAX(op2_min, op1_min),
3790 MIN(op2_max, op1_max));
3791 }
3792 }
3793 break;
3794 case ZEND_IS_SMALLER_OR_EQUAL:
3795 if (!exit_if_true) {
3796 /* op1 <= op2 */
3797 if (ssa_op->op1_use >= 0) {
3798 zend_jit_trace_set_var_range(
3799 &ssa->var_info[ssa_op->op1_use],
3800 op1_min,
3801 MIN(op1_max, op2_max));
3802 }
3803 if (ssa_op->op2_use >= 0) {
3804 zend_jit_trace_set_var_range(
3805 &ssa->var_info[ssa_op->op2_use],
3806 MAX(op2_min, op1_min),
3807 op2_max);
3808 }
3809 } else {
3810 /* op1 > op2 */
3811 if (ssa_op->op1_use >= 0) {
3812 zend_jit_trace_set_var_range(
3813 &ssa->var_info[ssa_op->op1_use],
3814 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3815 op1_max);
3816 }
3817 if (ssa_op->op2_use >= 0) {
3818 zend_jit_trace_set_var_range(
3819 &ssa->var_info[ssa_op->op2_use],
3820 op2_min,
3821 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3822 }
3823 }
3824 break;
3825 case ZEND_IS_SMALLER:
3826 if (!exit_if_true) {
3827 /* op1 < op2 */
3828 if (ssa_op->op1_use >= 0) {
3829 zend_jit_trace_set_var_range(
3830 &ssa->var_info[ssa_op->op1_use],
3831 op1_min,
3832 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3833 }
3834 if (ssa_op->op2_use >= 0) {
3835 zend_jit_trace_set_var_range(
3836 &ssa->var_info[ssa_op->op2_use],
3837 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3838 op2_max);
3839 }
3840 } else {
3841 /* op1 >= op2 */
3842 if (ssa_op->op1_use >= 0) {
3843 zend_jit_trace_set_var_range(
3844 &ssa->var_info[ssa_op->op1_use],
3845 MAX(op1_min, op2_min),
3846 op1_max);
3847 }
3848 if (ssa_op->op2_use >= 0) {
3849 zend_jit_trace_set_var_range(
3850 &ssa->var_info[ssa_op->op2_use],
3851 op2_min,
3852 MIN(op2_max, op1_max));
3853 }
3854 }
3855 break;
3856 }
3857 }
3858
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3859 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3860 {
3861 zend_uchar prev_opcode;
3862
3863 if (opline->op1_type == IS_CONST
3864 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3865 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3866 if (ssa_op->op2_use >= 0) {
3867 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3868 ssa_op--;
3869 opline = ssa_opcodes[ssa_op - ssa->ops];
3870 prev_opcode = opline->opcode;
3871 if (prev_opcode == ZEND_PRE_INC
3872 || prev_opcode == ZEND_PRE_DEC
3873 || prev_opcode == ZEND_POST_INC
3874 || prev_opcode == ZEND_POST_DEC) {
3875 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3876 }
3877 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3878 ssa_op--;
3879 opline = ssa_opcodes[ssa_op - ssa->ops];
3880 prev_opcode = opline->opcode;
3881 if (prev_opcode == ZEND_ADD
3882 || prev_opcode == ZEND_SUB) {
3883 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3884 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3885 }
3886 }
3887 }
3888 } else if (opline->op2_type == IS_CONST
3889 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3890 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3891 if (ssa_op->op1_use >= 0) {
3892 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3893 ssa_op--;
3894 opline = ssa_opcodes[ssa_op - ssa->ops];
3895 prev_opcode = opline->opcode;
3896 if (prev_opcode == ZEND_PRE_INC
3897 || prev_opcode == ZEND_PRE_DEC
3898 || prev_opcode == ZEND_POST_INC
3899 || prev_opcode == ZEND_POST_DEC) {
3900 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3901 }
3902 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3903 ssa_op--;
3904 opline = ssa_opcodes[ssa_op - ssa->ops];
3905 prev_opcode = opline->opcode;
3906 if (prev_opcode == ZEND_ADD
3907 || prev_opcode == ZEND_SUB) {
3908 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3909 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3910 }
3911 }
3912 }
3913 } else {
3914 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3915 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3916
3917 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3918 && prev_ssa_op != ssa->ops
3919 && prev_ssa_op->op1_use >= 0
3920 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3921 prev_ssa_op--;
3922 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3923 }
3924
3925 if (ssa_op->op1_use == prev_ssa_op->op1_use
3926 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3927 if (prev_opcode == ZEND_IS_EQUAL
3928 || prev_opcode == ZEND_IS_NOT_EQUAL
3929 || prev_opcode == ZEND_IS_SMALLER
3930 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3931 || prev_opcode == ZEND_CASE
3932 || prev_opcode == ZEND_IS_IDENTICAL
3933 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3934 || prev_opcode == ZEND_CASE_STRICT) {
3935 if (ssa_op->op1_use < 0) {
3936 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3937 return 0;
3938 }
3939 }
3940 if (ssa_op->op2_use < 0) {
3941 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3942 return 0;
3943 }
3944 }
3945 return 1;
3946 }
3947 }
3948 }
3949 return 0;
3950 }
3951
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3952 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3953 zend_jit_trace_rec *p,
3954 zend_jit_trace_stack_frame *frame)
3955 {
3956 if (opline->result_type == IS_TMP_VAR
3957 && (p+1)->op == ZEND_JIT_TRACE_VM
3958 && (p+1)->opline == opline + 1
3959 && ((opline+1)->opcode == ZEND_SEND_VAL
3960 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3961 && frame
3962 && frame->call
3963 && frame->call->func
3964 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3965 && (opline+1)->op1_type == IS_TMP_VAR
3966 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3967 && (opline+1)->op1.var == opline->result.var) {
3968
3969 if (frame->call && frame->call->func) {
3970 uint8_t res_type = (p+1)->op1_type;
3971
3972 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3973 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3974 }
3975 }
3976 return 1;
3977 }
3978 return 0;
3979 }
3980
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)3981 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
3982 {
3983 const void *handler = NULL;
3984 dasm_State* dasm_state = NULL;
3985 zend_script *script = NULL;
3986 zend_lifetime_interval **ra = NULL;
3987 zend_string *name = NULL;
3988 void *checkpoint;
3989 const zend_op_array *op_array;
3990 zend_ssa *ssa, *op_array_ssa;
3991 const zend_op **ssa_opcodes;
3992 zend_jit_trace_rec *p;
3993 zend_jit_op_array_trace_extension *jit_extension;
3994 int num_op_arrays = 0;
3995 zend_jit_trace_info *t;
3996 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
3997 zend_uchar smart_branch_opcode;
3998 const void *exit_addr;
3999 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info;
4000 bool send_result = 0;
4001 bool skip_comparison;
4002 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
4003 zend_class_entry *ce;
4004 bool ce_is_instanceof;
4005 bool on_this = 0;
4006 bool delayed_fetch_this = 0;
4007 bool avoid_refcounting = 0;
4008 bool polymorphic_side_trace =
4009 parent_trace &&
4010 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
4011 uint32_t i;
4012 zend_jit_trace_stack_frame *frame, *top, *call;
4013 zend_jit_trace_stack *stack;
4014 zend_uchar res_type = IS_UNKNOWN;
4015 const zend_op *opline, *orig_opline;
4016 const zend_ssa_op *ssa_op, *orig_ssa_op;
4017 int checked_stack;
4018 int peek_checked_stack;
4019 uint32_t frame_flags = 0;
4020
4021 JIT_G(current_trace) = trace_buffer;
4022
4023 checkpoint = zend_arena_checkpoint(CG(arena));
4024
4025 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4026
4027 if (!ssa) {
4028 goto jit_cleanup;
4029 }
4030
4031 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4032
4033 /* Register allocation */
4034 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4035 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4036 ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4037 }
4038
4039 p = trace_buffer;
4040 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4041 op_array = p->op_array;
4042 frame = JIT_G(current_frame);
4043 top = zend_jit_trace_call_frame(frame, op_array);
4044 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4045 frame->used_stack = checked_stack = peek_checked_stack = 0;
4046 stack = frame->stack;
4047 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4048 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4049 }
4050
4051 opline = p[1].opline;
4052 name = zend_jit_trace_name(op_array, opline->lineno);
4053 p += ZEND_JIT_TRACE_START_REC_SIZE;
4054
4055 dasm_init(&dasm_state, DASM_MAXSECTION);
4056 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4057 dasm_setup(&dasm_state, dasm_actions);
4058
4059 jit_extension =
4060 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4061 op_array_ssa = &jit_extension->func_info.ssa;
4062
4063 dasm_growpc(&dasm_state, 2); /* =>0: loop header */
4064 /* =>1: end of code */
4065
4066 zend_jit_align_func(&dasm_state);
4067 if (!parent_trace) {
4068 zend_jit_prologue(&dasm_state);
4069 }
4070 zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM,
4071 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4072
4073 if (!parent_trace) {
4074 zend_jit_set_last_valid_opline(opline);
4075 zend_jit_track_last_valid_opline();
4076 } else {
4077 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4078 zend_jit_trace_opline_guard(&dasm_state, opline);
4079 } else {
4080 zend_jit_reset_last_valid_opline();
4081 }
4082 }
4083
4084 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4085 int last_var;
4086 int parent_vars_count = 0;
4087 zend_jit_trace_stack *parent_stack = NULL;
4088 int used_stack = ((zend_tssa*)ssa)->used_stack;
4089
4090 if (used_stack > 0) {
4091 peek_checked_stack = used_stack;
4092 if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) {
4093 goto jit_failure;
4094 }
4095 }
4096
4097 if (parent_trace) {
4098 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4099 op_array->last_var + op_array->T);
4100 if (parent_vars_count) {
4101 parent_stack =
4102 zend_jit_traces[parent_trace].stack_map +
4103 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4104 }
4105 }
4106
4107 last_var = op_array->last_var;
4108 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4109 last_var += op_array->T;
4110 }
4111
4112 for (i = 0; i < last_var; i++) {
4113 uint32_t info = ssa->var_info[i].type;
4114
4115 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4116 uint8_t type, mem_type;
4117
4118 type = concrete_type(info);
4119 if (i < parent_vars_count
4120 && STACK_TYPE(parent_stack, i) == type) {
4121 mem_type = STACK_MEM_TYPE(parent_stack, i);
4122 if (mem_type != IS_UNKNOWN) {
4123 SET_STACK_TYPE(stack, i, mem_type, 1);
4124 }
4125 SET_STACK_TYPE(stack, i, type, 0);
4126 } else {
4127 SET_STACK_TYPE(stack, i, type, 1);
4128 }
4129 } else if (ssa->vars[i].alias != NO_ALIAS) {
4130 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4131 } else if (i < parent_vars_count
4132 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4133 /* This must be already handled by trace type inference */
4134 ZEND_UNREACHABLE();
4135 // SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
4136 } else if ((info & MAY_BE_GUARD) != 0
4137 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4138 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4139 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4140 && (opline-1)->result_type == IS_VAR
4141 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4142 && (ssa->vars[i].use_chain != -1
4143 || (ssa->vars[i].phi_use_chain
4144 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4145 /* Check loop-invariant variable type */
4146 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4147 goto jit_failure;
4148 }
4149 info &= ~MAY_BE_GUARD;
4150 ssa->var_info[i].type = info;
4151 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4152 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4153 && op_array->function_name
4154 && i >= op_array->num_args) {
4155 /* This must be already handled by trace type inference */
4156 ZEND_UNREACHABLE();
4157 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4158 }
4159
4160 if ((info & MAY_BE_PACKED_GUARD) != 0
4161 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4162 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4163 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4164 && (ssa->vars[i].use_chain != -1
4165 || (ssa->vars[i].phi_use_chain
4166 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4167 if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) {
4168 goto jit_failure;
4169 }
4170 info &= ~MAY_BE_PACKED_GUARD;
4171 ssa->var_info[i].type = info;
4172 }
4173 }
4174
4175 if (parent_trace) {
4176 /* Deoptimization */
4177 if (!zend_jit_trace_deoptimization(&dasm_state,
4178 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4179 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4180 parent_stack, parent_vars_count, ssa, stack, ra,
4181 polymorphic_side_trace)) {
4182 goto jit_failure;
4183 }
4184 }
4185
4186 if (ra
4187 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4188 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4189 for (i = 0; i < last_var; i++) {
4190 if (ra[i]
4191 && (ra[i]->flags & ZREG_LOAD) != 0
4192 && ra[i]->reg != stack[i].reg) {
4193
4194 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4195 uint8_t op_type;
4196
4197 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4198 op_type = concrete_type(ssa->var_info[i].type);
4199 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
4200 goto jit_failure;
4201 }
4202 SET_STACK_TYPE(stack, i, op_type, 1);
4203 }
4204
4205 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4206 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4207 goto jit_failure;
4208 }
4209 }
4210 }
4211 }
4212 }
4213
4214 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4215 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4216 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4217
4218 zend_jit_label(&dasm_state, 0); /* start of of trace loop */
4219
4220 if (ra) {
4221 zend_ssa_phi *phi = ssa->blocks[1].phis;
4222
4223 while (phi) {
4224 zend_lifetime_interval *ival = ra[phi->ssa_var];
4225
4226 if (ival) {
4227 if (ival->flags & ZREG_LOAD) {
4228 uint32_t info = ssa->var_info[phi->ssa_var].type;
4229 ZEND_ASSERT(ival->reg != ZREG_NONE);
4230
4231 if (info & MAY_BE_GUARD) {
4232 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4233 goto jit_failure;
4234 }
4235 info &= ~MAY_BE_GUARD;
4236 ssa->var_info[phi->ssa_var].type = info;
4237 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4238 }
4239 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_LOAD);
4240 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
4241 goto jit_failure;
4242 }
4243 } else if (ival->flags & ZREG_STORE) {
4244 ZEND_ASSERT(ival->reg != ZREG_NONE);
4245
4246 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_STORE);
4247 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg,
4248 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4249 goto jit_failure;
4250 }
4251 } else {
4252 /* Register has to be written back on side exit */
4253 SET_STACK_REG(stack, phi->var, ival->reg);
4254 }
4255 }
4256 phi = phi->next;
4257 }
4258 }
4259
4260 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4261 // if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4262 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4263 //
4264 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4265 // if (!timeout_exit_addr) {
4266 // goto jit_failure;
4267 // }
4268 // }
4269 // }
4270
4271 if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
4272 int last_var = op_array->last_var;
4273
4274 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4275 last_var += op_array->T;
4276 }
4277 for (i = 0; i < last_var; i++) {
4278 if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) {
4279 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4280 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4281 goto jit_failure;
4282 }
4283 }
4284 }
4285 }
4286 }
4287
4288 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4289 for (;;p++) {
4290 if (p->op == ZEND_JIT_TRACE_VM) {
4291 uint8_t op1_type = p->op1_type;
4292 uint8_t op2_type = p->op2_type;
4293 uint8_t op3_type = p->op3_type;
4294 uint8_t orig_op1_type = op1_type;
4295 uint8_t orig_op2_type = op2_type;
4296 uint8_t val_type = IS_UNKNOWN;
4297 bool op1_indirect;
4298 zend_class_entry *op1_ce = NULL;
4299 zend_class_entry *op2_ce = NULL;
4300 bool gen_handler;
4301
4302 opline = p->opline;
4303 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4304 op1_type = IS_UNKNOWN;
4305 }
4306 if (op1_type != IS_UNKNOWN) {
4307 op1_type &= ~IS_TRACE_PACKED;
4308 }
4309 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4310 op2_type = IS_UNKNOWN;
4311 }
4312 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4313 op3_type = IS_UNKNOWN;
4314 }
4315
4316 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4317 op1_ce = (zend_class_entry*)(p+1)->ce;
4318 p++;
4319 }
4320 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4321 op2_ce = (zend_class_entry*)(p+1)->ce;
4322 p++;
4323 }
4324 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4325 val_type = (p+1)->op1_type;
4326 p++;
4327 }
4328
4329 frame_flags = 0;
4330
4331 switch (opline->opcode) {
4332 case ZEND_INIT_FCALL:
4333 case ZEND_INIT_FCALL_BY_NAME:
4334 case ZEND_INIT_NS_FCALL_BY_NAME:
4335 case ZEND_INIT_METHOD_CALL:
4336 case ZEND_INIT_DYNAMIC_CALL:
4337 case ZEND_INIT_STATIC_METHOD_CALL:
4338 case ZEND_INIT_USER_CALL:
4339 case ZEND_NEW:
4340 frame->call_level++;
4341 }
4342
4343 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4344 gen_handler = 0;
4345 switch (opline->opcode) {
4346 case ZEND_PRE_INC:
4347 case ZEND_PRE_DEC:
4348 case ZEND_POST_INC:
4349 case ZEND_POST_DEC:
4350 if (opline->op1_type != IS_CV) {
4351 break;
4352 }
4353 op1_info = OP1_INFO();
4354 CHECK_OP1_TRACE_TYPE();
4355 if (!(op1_info & MAY_BE_LONG)) {
4356 break;
4357 }
4358 if (opline->result_type != IS_UNUSED) {
4359 res_use_info = zend_jit_trace_type_to_info(
4360 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4361 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4362 res_info = RES_INFO();
4363 res_addr = RES_REG_ADDR();
4364 } else {
4365 res_use_info = -1;
4366 res_info = -1;
4367 res_addr = 0;
4368 }
4369 op1_def_info = OP1_DEF_INFO();
4370 if (op1_def_info & MAY_BE_GUARD
4371 && !has_concrete_type(op1_def_info)) {
4372 op1_def_info &= ~MAY_BE_GUARD;
4373 }
4374 if (!zend_jit_inc_dec(&dasm_state, opline,
4375 op1_info, OP1_REG_ADDR(),
4376 op1_def_info, OP1_DEF_REG_ADDR(),
4377 res_use_info, res_info,
4378 res_addr,
4379 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4380 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4381 goto jit_failure;
4382 }
4383 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4384 && !(op1_info & MAY_BE_STRING)) {
4385 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4386 if (opline->result_type != IS_UNUSED) {
4387 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4388 }
4389 }
4390 if (opline->result_type != IS_UNUSED
4391 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4392 && !(op1_info & MAY_BE_STRING)) {
4393 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4394 }
4395 goto done;
4396 case ZEND_BW_OR:
4397 case ZEND_BW_AND:
4398 case ZEND_BW_XOR:
4399 case ZEND_SL:
4400 case ZEND_SR:
4401 case ZEND_MOD:
4402 op1_info = OP1_INFO();
4403 CHECK_OP1_TRACE_TYPE();
4404 op2_info = OP2_INFO();
4405 CHECK_OP2_TRACE_TYPE();
4406 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4407 break;
4408 }
4409 if (!(op1_info & MAY_BE_LONG)
4410 || !(op2_info & MAY_BE_LONG)) {
4411 break;
4412 }
4413 res_addr = RES_REG_ADDR();
4414 if (Z_MODE(res_addr) != IS_REG
4415 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4416 send_result = 1;
4417 res_use_info = -1;
4418 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4419 if (!zend_jit_reuse_ip(&dasm_state)) {
4420 goto jit_failure;
4421 }
4422 } else {
4423 res_use_info = zend_jit_trace_type_to_info(
4424 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4425 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4426 }
4427 res_info = RES_INFO();
4428 if (!zend_jit_long_math(&dasm_state, opline,
4429 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4430 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4431 res_use_info, res_info, res_addr,
4432 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4433 goto jit_failure;
4434 }
4435 goto done;
4436 case ZEND_ADD:
4437 case ZEND_SUB:
4438 case ZEND_MUL:
4439 // case ZEND_DIV: // TODO: check for division by zero ???
4440 op1_info = OP1_INFO();
4441 op1_addr = OP1_REG_ADDR();
4442 op2_info = OP2_INFO();
4443 op2_addr = OP2_REG_ADDR();
4444 if (orig_op1_type != IS_UNKNOWN
4445 && (orig_op1_type & IS_TRACE_REFERENCE)
4446 && opline->op1_type == IS_CV
4447 && (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_FCARG1)
4448 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4449 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4450 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4451 goto jit_failure;
4452 }
4453 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4454 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4455 }
4456 } else {
4457 CHECK_OP1_TRACE_TYPE();
4458 }
4459 if (orig_op2_type != IS_UNKNOWN
4460 && (orig_op2_type & IS_TRACE_REFERENCE)
4461 && opline->op2_type == IS_CV
4462 && (Z_MODE(op1_addr) != IS_REG || Z_REG(op1_addr) != ZREG_FCARG1)
4463 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4464 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op2_type, &op2_info, &op2_addr,
4465 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4466 goto jit_failure;
4467 }
4468 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4469 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4470 }
4471 } else {
4472 CHECK_OP2_TRACE_TYPE();
4473 }
4474 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4475 break;
4476 }
4477 if (opline->opcode == ZEND_ADD &&
4478 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4479 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4480 /* pass */
4481 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4482 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4483 break;
4484 }
4485 res_addr = RES_REG_ADDR();
4486 if (Z_MODE(res_addr) != IS_REG
4487 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4488 send_result = 1;
4489 res_use_info = -1;
4490 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4491 if (!zend_jit_reuse_ip(&dasm_state)) {
4492 goto jit_failure;
4493 }
4494 } else {
4495 res_use_info = zend_jit_trace_type_to_info(
4496 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4497 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4498 }
4499 res_info = RES_INFO();
4500 if (opline->opcode == ZEND_ADD &&
4501 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4502 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4503 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4504 goto jit_failure;
4505 }
4506 } else {
4507 if (!zend_jit_math(&dasm_state, opline,
4508 op1_info, op1_addr,
4509 op2_info, op2_addr,
4510 res_use_info, res_info, res_addr,
4511 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4512 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4513 goto jit_failure;
4514 }
4515 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4516 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4517 && has_concrete_type(op1_info)
4518 && has_concrete_type(op2_info)) {
4519 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4520 }
4521 }
4522 goto done;
4523 case ZEND_CONCAT:
4524 case ZEND_FAST_CONCAT:
4525 op1_info = OP1_INFO();
4526 CHECK_OP1_TRACE_TYPE();
4527 op2_info = OP2_INFO();
4528 CHECK_OP2_TRACE_TYPE();
4529 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4530 break;
4531 }
4532 if (!(op1_info & MAY_BE_STRING) ||
4533 !(op2_info & MAY_BE_STRING)) {
4534 break;
4535 }
4536 res_addr = RES_REG_ADDR();
4537 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4538 send_result = 1;
4539 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4540 if (!zend_jit_reuse_ip(&dasm_state)) {
4541 goto jit_failure;
4542 }
4543 }
4544 if (!zend_jit_concat(&dasm_state, opline,
4545 op1_info, op2_info, res_addr,
4546 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4547 goto jit_failure;
4548 }
4549 goto done;
4550 case ZEND_ASSIGN_OP:
4551 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4552 break;
4553 }
4554 op1_info = OP1_INFO();
4555 CHECK_OP1_TRACE_TYPE();
4556 op2_info = OP2_INFO();
4557 CHECK_OP2_TRACE_TYPE();
4558 if (!zend_jit_supported_binary_op(
4559 opline->extended_value, op1_info, op2_info)) {
4560 break;
4561 }
4562 op1_def_info = OP1_DEF_INFO();
4563 if (op1_def_info & MAY_BE_GUARD
4564 && !has_concrete_type(op1_def_info)) {
4565 op1_def_info &= ~MAY_BE_GUARD;
4566 }
4567 if (!zend_jit_assign_op(&dasm_state, opline,
4568 op1_info, op1_def_info, OP1_RANGE(),
4569 op2_info, OP2_RANGE(),
4570 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4571 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4572 goto jit_failure;
4573 }
4574 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4575 && has_concrete_type(op1_info)
4576 && has_concrete_type(op2_info)) {
4577 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4578 if (opline->result_type != IS_UNUSED) {
4579 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4580 }
4581 }
4582 goto done;
4583 case ZEND_ASSIGN_DIM_OP:
4584 if (opline->result_type != IS_UNUSED) {
4585 break;
4586 }
4587 if (!zend_jit_supported_binary_op(
4588 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4589 break;
4590 }
4591 if (opline->op1_type == IS_CV
4592 && (opline+1)->op1_type == IS_CV
4593 && (opline+1)->op1.var == opline->op1.var) {
4594 /* skip $a[x] += $a; */
4595 break;
4596 }
4597 op1_info = OP1_INFO();
4598 op1_addr = OP1_REG_ADDR();
4599 if (opline->op1_type == IS_VAR) {
4600 if (orig_op1_type != IS_UNKNOWN
4601 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4602 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4603 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4604 goto jit_failure;
4605 }
4606 } else {
4607 break;
4608 }
4609 }
4610 if (orig_op1_type != IS_UNKNOWN
4611 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4612 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4613 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4614 goto jit_failure;
4615 }
4616 if (opline->op1_type == IS_CV
4617 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4618 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4619 }
4620 } else {
4621 CHECK_OP1_TRACE_TYPE();
4622 }
4623 op2_info = OP2_INFO();
4624 CHECK_OP2_TRACE_TYPE();
4625 op1_data_info = OP1_DATA_INFO();
4626 CHECK_OP1_DATA_TRACE_TYPE();
4627 op1_def_info = OP1_DEF_INFO();
4628 if (!zend_jit_assign_dim_op(&dasm_state, opline,
4629 op1_info, op1_def_info, op1_addr, op2_info,
4630 op1_data_info, OP1_DATA_RANGE(), val_type,
4631 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4632 goto jit_failure;
4633 }
4634 goto done;
4635 case ZEND_PRE_INC_OBJ:
4636 case ZEND_PRE_DEC_OBJ:
4637 case ZEND_POST_INC_OBJ:
4638 case ZEND_POST_DEC_OBJ:
4639 if (opline->op2_type != IS_CONST
4640 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4641 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4642 break;
4643 }
4644 ce = NULL;
4645 ce_is_instanceof = 0;
4646 on_this = delayed_fetch_this = 0;
4647 op1_indirect = 0;
4648 if (opline->op1_type == IS_UNUSED) {
4649 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4650 ce = op_array->scope;
4651 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4652 op1_addr = 0;
4653 on_this = 1;
4654 } else {
4655 if (ssa_op->op1_use >= 0) {
4656 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4657 }
4658 op1_info = OP1_INFO();
4659 if (!(op1_info & MAY_BE_OBJECT)) {
4660 break;
4661 }
4662 op1_addr = OP1_REG_ADDR();
4663 if (opline->op1_type == IS_VAR) {
4664 if (orig_op1_type != IS_UNKNOWN
4665 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4666 op1_indirect = 1;
4667 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4668 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4669 goto jit_failure;
4670 }
4671 }
4672 }
4673 if (orig_op1_type != IS_UNKNOWN
4674 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4675 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4676 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4677 goto jit_failure;
4678 }
4679 if (opline->op1_type == IS_CV
4680 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4681 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4682 }
4683 } else {
4684 CHECK_OP1_TRACE_TYPE();
4685 }
4686 if (!(op1_info & MAY_BE_OBJECT)) {
4687 break;
4688 }
4689 if (ssa->var_info && ssa->ops) {
4690 if (ssa_op->op1_use >= 0) {
4691 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4692 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4693 ce = op1_ssa->ce;
4694 ce_is_instanceof = op1_ssa->is_instanceof;
4695 }
4696 }
4697 }
4698 if (delayed_fetch_this) {
4699 on_this = 1;
4700 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4701 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4702 } else if (op_array_ssa->ops
4703 && op_array_ssa->vars
4704 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4705 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4706 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4707 }
4708 }
4709 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4710 op1_info, op1_addr,
4711 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4712 val_type)) {
4713 goto jit_failure;
4714 }
4715 goto done;
4716 case ZEND_ASSIGN_OBJ_OP:
4717 if (opline->result_type != IS_UNUSED) {
4718 break;
4719 }
4720 if (opline->op2_type != IS_CONST
4721 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4722 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4723 break;
4724 }
4725 if (opline->op1_type == IS_CV
4726 && (opline+1)->op1_type == IS_CV
4727 && (opline+1)->op1.var == opline->op1.var) {
4728 /* skip $a->prop += $a; */
4729 break;
4730 }
4731 if (!zend_jit_supported_binary_op(
4732 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4733 break;
4734 }
4735 ce = NULL;
4736 ce_is_instanceof = 0;
4737 on_this = delayed_fetch_this = 0;
4738 op1_indirect = 0;
4739 if (opline->op1_type == IS_UNUSED) {
4740 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4741 ce = op_array->scope;
4742 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4743 op1_addr = 0;
4744 on_this = 1;
4745 } else {
4746 if (ssa_op->op1_use >= 0) {
4747 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4748 }
4749 op1_info = OP1_INFO();
4750 if (!(op1_info & MAY_BE_OBJECT)) {
4751 break;
4752 }
4753 op1_addr = OP1_REG_ADDR();
4754 if (opline->op1_type == IS_VAR) {
4755 if (orig_op1_type != IS_UNKNOWN
4756 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4757 op1_indirect = 1;
4758 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4759 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4760 goto jit_failure;
4761 }
4762 }
4763 }
4764 if (orig_op1_type != IS_UNKNOWN
4765 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4766 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4767 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4768 goto jit_failure;
4769 }
4770 if (opline->op1_type == IS_CV
4771 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4772 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4773 }
4774 } else {
4775 CHECK_OP1_TRACE_TYPE();
4776 }
4777 if (!(op1_info & MAY_BE_OBJECT)) {
4778 break;
4779 }
4780 if (ssa->var_info && ssa->ops) {
4781 if (ssa_op->op1_use >= 0) {
4782 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4783 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4784 ce = op1_ssa->ce;
4785 ce_is_instanceof = op1_ssa->is_instanceof;
4786 }
4787 }
4788 }
4789 if (delayed_fetch_this) {
4790 on_this = 1;
4791 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4792 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4793 } else if (op_array_ssa->ops
4794 && op_array_ssa->vars
4795 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4796 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4797 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4798 }
4799 }
4800 op1_data_info = OP1_DATA_INFO();
4801 CHECK_OP1_DATA_TRACE_TYPE();
4802 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
4803 op1_info, op1_addr, op1_data_info, OP1_DATA_RANGE(),
4804 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4805 val_type)) {
4806 goto jit_failure;
4807 }
4808 goto done;
4809 case ZEND_ASSIGN_OBJ:
4810 if (opline->op2_type != IS_CONST
4811 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4812 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4813 break;
4814 }
4815 ce = NULL;
4816 ce_is_instanceof = 0;
4817 on_this = delayed_fetch_this = 0;
4818 op1_indirect = 0;
4819 if (opline->op1_type == IS_UNUSED) {
4820 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4821 ce = op_array->scope;
4822 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4823 op1_addr = 0;
4824 on_this = 1;
4825 } else {
4826 if (ssa_op->op1_use >= 0) {
4827 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4828 }
4829 op1_info = OP1_INFO();
4830 if (!(op1_info & MAY_BE_OBJECT)) {
4831 break;
4832 }
4833 op1_addr = OP1_REG_ADDR();
4834 if (opline->op1_type == IS_VAR) {
4835 if (orig_op1_type != IS_UNKNOWN
4836 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4837 op1_indirect = 1;
4838 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4839 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4840 goto jit_failure;
4841 }
4842 }
4843 }
4844 if (orig_op1_type != IS_UNKNOWN
4845 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4846 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4847 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4848 goto jit_failure;
4849 }
4850 if (opline->op1_type == IS_CV
4851 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4852 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4853 }
4854 } else {
4855 CHECK_OP1_TRACE_TYPE();
4856 }
4857 if (!(op1_info & MAY_BE_OBJECT)) {
4858 break;
4859 }
4860 if (ssa->var_info && ssa->ops) {
4861 if (ssa_op->op1_use >= 0) {
4862 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4863 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4864 ce = op1_ssa->ce;
4865 ce_is_instanceof = op1_ssa->is_instanceof;
4866 }
4867 }
4868 }
4869 if (delayed_fetch_this) {
4870 on_this = 1;
4871 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4872 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4873 } else if (op_array_ssa->ops
4874 && op_array_ssa->vars
4875 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4876 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4877 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4878 }
4879 }
4880 op1_data_info = OP1_DATA_INFO();
4881 CHECK_OP1_DATA_TRACE_TYPE();
4882 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4883 op1_info, op1_addr, op1_data_info,
4884 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4885 val_type,
4886 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4887 goto jit_failure;
4888 }
4889 if ((opline+1)->op1_type == IS_CV
4890 && (ssa_op+1)->op1_def >= 0
4891 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4892 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4893 }
4894 goto done;
4895 case ZEND_ASSIGN_DIM:
4896 op1_info = OP1_INFO();
4897 op1_addr = OP1_REG_ADDR();
4898 if (opline->op1_type == IS_CV
4899 && (opline+1)->op1_type == IS_CV
4900 && (opline+1)->op1.var == opline->op1.var) {
4901 /* skip $a[x] = $a; */
4902 break;
4903 }
4904 if (opline->op1_type == IS_VAR) {
4905 if (orig_op1_type != IS_UNKNOWN
4906 && (orig_op1_type & IS_TRACE_INDIRECT)
4907 && opline->result_type == IS_UNUSED) {
4908 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4909 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4910 goto jit_failure;
4911 }
4912 } else {
4913 break;
4914 }
4915 }
4916 if (orig_op1_type != IS_UNKNOWN
4917 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4918 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4919 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4920 goto jit_failure;
4921 }
4922 if (opline->op1_type == IS_CV
4923 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4924 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4925 }
4926 } else {
4927 CHECK_OP1_TRACE_TYPE();
4928 }
4929 op2_info = OP2_INFO();
4930 CHECK_OP2_TRACE_TYPE();
4931 op1_data_info = OP1_DATA_INFO();
4932 CHECK_OP1_DATA_TRACE_TYPE();
4933 if (!zend_jit_assign_dim(&dasm_state, opline,
4934 op1_info, op1_addr, op2_info, op1_data_info, val_type,
4935 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4936 goto jit_failure;
4937 }
4938 if ((opline+1)->op1_type == IS_CV
4939 && (ssa_op+1)->op1_def >= 0
4940 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4941 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4942 }
4943 goto done;
4944 case ZEND_ASSIGN:
4945 if (opline->op1_type != IS_CV) {
4946 break;
4947 }
4948 op2_addr = OP2_REG_ADDR();
4949 op2_info = OP2_INFO();
4950
4951 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
4952 op2_def_addr = op2_addr;
4953 } else {
4954 op2_def_addr = OP2_DEF_REG_ADDR();
4955 }
4956 CHECK_OP2_TRACE_TYPE();
4957 op1_info = OP1_INFO();
4958 op1_def_info = OP1_DEF_INFO();
4959 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
4960 if (op1_type < IS_STRING
4961 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
4962 if (!zend_jit_scalar_type_guard(&dasm_state, opline, opline->op1.var)) {
4963 goto jit_failure;
4964 }
4965 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
4966 } else {
4967 CHECK_OP1_TRACE_TYPE();
4968 }
4969 }
4970 op1_addr = OP1_REG_ADDR();
4971 op1_def_addr = OP1_DEF_REG_ADDR();
4972 if (Z_MODE(op1_def_addr) != IS_REG &&
4973 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
4974 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
4975 /* type may be not set */
4976 op1_info |= MAY_BE_NULL;
4977 }
4978 if (orig_op1_type != IS_UNKNOWN) {
4979 if (orig_op1_type & IS_TRACE_REFERENCE) {
4980 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4981 !ssa->var_info[ssa_op->op1_use].guarded_reference, 0)) {
4982 goto jit_failure;
4983 }
4984 if (opline->op1_type == IS_CV
4985 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4986 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4987 }
4988 if (opline->result_type == IS_UNUSED) {
4989 res_addr = 0;
4990 } else {
4991 res_addr = RES_REG_ADDR();
4992 if (Z_MODE(res_addr) != IS_REG
4993 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4994 send_result = 1;
4995 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4996 if (!zend_jit_reuse_ip(&dasm_state)) {
4997 goto jit_failure;
4998 }
4999 }
5000 }
5001 if (!zend_jit_assign_to_typed_ref(&dasm_state, opline, opline->op2_type, op2_addr, res_addr, 1)) {
5002 goto jit_failure;
5003 }
5004 op1_def_addr = op1_addr;
5005 op1_def_info &= ~MAY_BE_REF;
5006 } else if (op1_info & MAY_BE_REF) {
5007 if (!zend_jit_noref_guard(&dasm_state, opline, op1_addr)) {
5008 goto jit_failure;
5009 }
5010 op1_info &= ~MAY_BE_REF;
5011 op1_def_info &= ~MAY_BE_REF;
5012 }
5013 }
5014 if (opline->result_type == IS_UNUSED) {
5015 res_addr = 0;
5016 res_info = -1;
5017 } else {
5018 res_addr = RES_REG_ADDR();
5019 res_info = RES_INFO();
5020 if (Z_MODE(res_addr) != IS_REG
5021 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5022 send_result = 1;
5023 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5024 if (!zend_jit_reuse_ip(&dasm_state)) {
5025 goto jit_failure;
5026 }
5027 }
5028 }
5029 if (!zend_jit_assign(&dasm_state, opline,
5030 op1_info, op1_addr,
5031 op1_def_info, op1_def_addr,
5032 op2_info, op2_addr, op2_def_addr,
5033 res_info, res_addr,
5034 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5035 goto jit_failure;
5036 }
5037 if (ssa_op->op2_def >= 0
5038 && Z_MODE(op2_addr) == IS_REG
5039 && ssa->vars[ssa_op->op2_def].no_val) {
5040 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5041 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5042
5043 if (STACK_MEM_TYPE(stack, var_num) != type
5044 && ssa->vars[ssa_op->op2_def].use_chain < 0
5045 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5046 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5047 return 0;
5048 }
5049 SET_STACK_TYPE(stack, var_num, type, 1);
5050 }
5051 }
5052 if (opline->op2_type == IS_CV
5053 && ssa_op->op2_def >= 0
5054 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5055 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5056 }
5057 goto done;
5058 case ZEND_CAST:
5059 if (opline->extended_value != op1_type) {
5060 break;
5061 }
5062 ZEND_FALLTHROUGH;
5063 case ZEND_QM_ASSIGN:
5064 op1_addr = OP1_REG_ADDR();
5065 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5066 op1_def_addr = op1_addr;
5067 } else {
5068 op1_def_addr = OP1_DEF_REG_ADDR();
5069 }
5070 op1_info = OP1_INFO();
5071 CHECK_OP1_TRACE_TYPE();
5072 res_info = RES_INFO();
5073 res_use_info = zend_jit_trace_type_to_info(
5074 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
5075 res_addr = RES_REG_ADDR();
5076 if (Z_MODE(res_addr) != IS_REG &&
5077 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5078 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5079 /* type may be not set */
5080 res_use_info |= MAY_BE_NULL;
5081 }
5082 if (!zend_jit_qm_assign(&dasm_state, opline,
5083 op1_info, op1_addr, op1_def_addr,
5084 res_use_info, res_info, res_addr)) {
5085 goto jit_failure;
5086 }
5087 if (ssa_op->op1_def >= 0
5088 && Z_MODE(op1_addr) == IS_REG
5089 && ssa->vars[ssa_op->op1_def].no_val) {
5090 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5091 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5092
5093 if (STACK_MEM_TYPE(stack, var_num) != type
5094 && ssa->vars[ssa_op->op1_def].use_chain < 0
5095 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5096 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5097 return 0;
5098 }
5099 SET_STACK_TYPE(stack, var_num, type, 1);
5100 }
5101 }
5102 if (opline->op1_type == IS_CV
5103 && ssa_op->op1_def >= 0
5104 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5105 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5106 }
5107 goto done;
5108 case ZEND_INIT_FCALL:
5109 case ZEND_INIT_FCALL_BY_NAME:
5110 case ZEND_INIT_NS_FCALL_BY_NAME:
5111 frame_flags = TRACE_FRAME_MASK_NESTED;
5112 if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5113 goto jit_failure;
5114 }
5115 goto done;
5116 case ZEND_SEND_VAL:
5117 case ZEND_SEND_VAL_EX:
5118 if (opline->op2_type == IS_CONST) {
5119 /* Named parameters not supported in JIT */
5120 break;
5121 }
5122 if (opline->opcode == ZEND_SEND_VAL_EX
5123 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5124 break;
5125 }
5126 op1_info = OP1_INFO();
5127 CHECK_OP1_TRACE_TYPE();
5128 if (!zend_jit_send_val(&dasm_state, opline,
5129 op1_info, OP1_REG_ADDR())) {
5130 goto jit_failure;
5131 }
5132 if (frame->call && frame->call->func) {
5133 if (opline->op1_type == IS_CONST) {
5134 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5135 } else if (op1_type != IS_UNKNOWN) {
5136 if (op1_type == IS_UNDEF) {
5137 op1_type = IS_NULL;
5138 }
5139 zend_jit_trace_send_type(opline, frame->call, op1_type);
5140 }
5141 }
5142 goto done;
5143 case ZEND_SEND_REF:
5144 if (opline->op2_type == IS_CONST) {
5145 /* Named parameters not supported in JIT */
5146 break;
5147 }
5148 op1_info = OP1_INFO();
5149 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
5150 op1_info, 0)) {
5151 goto jit_failure;
5152 }
5153 if (opline->op1_type == IS_CV
5154 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5155 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5156 }
5157 goto done;
5158 case ZEND_SEND_VAR:
5159 case ZEND_SEND_VAR_EX:
5160 case ZEND_SEND_VAR_NO_REF:
5161 case ZEND_SEND_VAR_NO_REF_EX:
5162 case ZEND_SEND_FUNC_ARG:
5163 if (opline->op2_type == IS_CONST) {
5164 /* Named parameters not supported in JIT */
5165 break;
5166 }
5167 if ((opline->opcode == ZEND_SEND_VAR_EX
5168 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5169 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5170 break;
5171 }
5172 op1_addr = OP1_REG_ADDR();
5173 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5174 op1_def_addr = op1_addr;
5175 } else {
5176 op1_def_addr = OP1_DEF_REG_ADDR();
5177 }
5178 op1_info = OP1_INFO();
5179 CHECK_OP1_TRACE_TYPE();
5180 if (!zend_jit_send_var(&dasm_state, opline, op_array,
5181 op1_info, op1_addr, op1_def_addr)) {
5182 goto jit_failure;
5183 }
5184 if (ssa_op->op1_def >= 0
5185 && Z_MODE(op1_addr) == IS_REG
5186 && ssa->vars[ssa_op->op1_def].no_val) {
5187 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5188 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5189
5190 if (STACK_MEM_TYPE(stack, var_num) != type
5191 && ssa->vars[ssa_op->op1_def].use_chain < 0
5192 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5193 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5194 return 0;
5195 }
5196 SET_STACK_TYPE(stack, var_num, type, 1);
5197 }
5198 }
5199 if (opline->op1_type == IS_CV
5200 && ssa_op->op1_def >= 0
5201 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5202 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5203 }
5204 if (frame->call && frame->call->func) {
5205 if ((opline->opcode == ZEND_SEND_VAR_EX
5206 || opline->opcode == ZEND_SEND_FUNC_ARG)
5207 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5208 goto done;
5209 }
5210 if (op1_type != IS_UNKNOWN) {
5211 if (op1_type == IS_UNDEF) {
5212 op1_type = IS_NULL;
5213 }
5214 zend_jit_trace_send_type(opline, frame->call, op1_type);
5215 }
5216 }
5217 goto done;
5218 case ZEND_CHECK_FUNC_ARG:
5219 if (!JIT_G(current_frame)
5220 || !JIT_G(current_frame)->call
5221 || !JIT_G(current_frame)->call->func) {
5222 break;
5223 }
5224 if (opline->op2_type == IS_CONST
5225 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5226 /* Named parameters not supported in JIT */
5227 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5228 break;
5229 }
5230 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
5231 goto jit_failure;
5232 }
5233 goto done;
5234 case ZEND_CHECK_UNDEF_ARGS:
5235 if (JIT_G(current_frame)
5236 && JIT_G(current_frame)->call) {
5237 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5238 }
5239 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
5240 goto jit_failure;
5241 }
5242 goto done;
5243 case ZEND_DO_UCALL:
5244 case ZEND_DO_ICALL:
5245 case ZEND_DO_FCALL_BY_NAME:
5246 case ZEND_DO_FCALL:
5247 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5248 goto jit_failure;
5249 }
5250 goto done;
5251 case ZEND_IS_EQUAL:
5252 case ZEND_IS_NOT_EQUAL:
5253 case ZEND_IS_SMALLER:
5254 case ZEND_IS_SMALLER_OR_EQUAL:
5255 case ZEND_CASE:
5256 op1_info = OP1_INFO();
5257 op2_info = OP2_INFO();
5258 skip_comparison =
5259 ssa_op != ssa->ops &&
5260 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5261 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5262 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5263 CHECK_OP1_TRACE_TYPE();
5264 CHECK_OP2_TRACE_TYPE();
5265 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5266 bool exit_if_true = 0;
5267 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5268 uint32_t exit_point;
5269
5270 if (ra) {
5271 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5272 }
5273 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5274 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5275 if (!exit_addr) {
5276 goto jit_failure;
5277 }
5278 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5279 if (!zend_jit_cmp(&dasm_state, opline,
5280 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5281 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5282 RES_REG_ADDR(),
5283 zend_may_throw(opline, ssa_op, op_array, ssa),
5284 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5285 goto jit_failure;
5286 }
5287 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5288 } else {
5289 smart_branch_opcode = 0;
5290 exit_addr = NULL;
5291 if (!zend_jit_cmp(&dasm_state, opline,
5292 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5293 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5294 RES_REG_ADDR(),
5295 zend_may_throw(opline, ssa_op, op_array, ssa),
5296 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5297 goto jit_failure;
5298 }
5299 }
5300 goto done;
5301 case ZEND_IS_IDENTICAL:
5302 case ZEND_IS_NOT_IDENTICAL:
5303 case ZEND_CASE_STRICT:
5304 op1_info = OP1_INFO();
5305 op2_info = OP2_INFO();
5306 skip_comparison =
5307 ssa_op != ssa->ops &&
5308 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5309 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5310 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5311 CHECK_OP1_TRACE_TYPE();
5312 CHECK_OP2_TRACE_TYPE();
5313 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5314 bool exit_if_true = 0;
5315 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5316 uint32_t exit_point;
5317
5318 if (ra) {
5319 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5320 }
5321 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5322 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5323 if (!exit_addr) {
5324 goto jit_failure;
5325 }
5326 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5327 exit_if_true = !exit_if_true;
5328 }
5329 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5330 if (!zend_jit_identical(&dasm_state, opline,
5331 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5332 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5333 RES_REG_ADDR(),
5334 zend_may_throw(opline, ssa_op, op_array, ssa),
5335 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5336 goto jit_failure;
5337 }
5338 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5339 } else {
5340 smart_branch_opcode = 0;
5341 exit_addr = NULL;
5342 if (!zend_jit_identical(&dasm_state, opline,
5343 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5344 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5345 RES_REG_ADDR(),
5346 zend_may_throw(opline, ssa_op, op_array, ssa),
5347 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5348 goto jit_failure;
5349 }
5350 }
5351 goto done;
5352 case ZEND_DEFINED:
5353 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5354 bool exit_if_true = 0;
5355 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5356 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5357
5358 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5359 if (!exit_addr) {
5360 goto jit_failure;
5361 }
5362 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5363 } else {
5364 smart_branch_opcode = 0;
5365 exit_addr = NULL;
5366 }
5367 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5368 goto jit_failure;
5369 }
5370 goto done;
5371 case ZEND_TYPE_CHECK:
5372 if (opline->extended_value == MAY_BE_RESOURCE) {
5373 // TODO: support for is_resource() ???
5374 break;
5375 }
5376 op1_info = OP1_INFO();
5377 CHECK_OP1_TRACE_TYPE();
5378 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5379 bool exit_if_true = 0;
5380 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5381 uint32_t exit_point;
5382
5383 if (ra) {
5384 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5385 }
5386 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5387 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5388 if (!exit_addr) {
5389 goto jit_failure;
5390 }
5391 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5392 } else {
5393 smart_branch_opcode = 0;
5394 exit_addr = NULL;
5395 }
5396 if (!zend_jit_type_check(&dasm_state, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5397 goto jit_failure;
5398 }
5399 goto done;
5400 case ZEND_RETURN:
5401 op1_info = OP1_INFO();
5402 CHECK_OP1_TRACE_TYPE();
5403 if (opline->op1_type == IS_CONST) {
5404 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5405 } else if (op1_type != IS_UNKNOWN) {
5406 res_type = op1_type;
5407 }
5408 if (op_array->type == ZEND_EVAL_CODE
5409 // TODO: support for top-level code
5410 || !op_array->function_name
5411 // TODO: support for IS_UNDEF ???
5412 || (op1_info & MAY_BE_UNDEF)) {
5413 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5414 goto jit_failure;
5415 }
5416 } else {
5417 int j;
5418 int may_throw = 0;
5419 bool left_frame = 0;
5420
5421 if (!zend_jit_return(&dasm_state, opline, op_array,
5422 op1_info, OP1_REG_ADDR())) {
5423 goto jit_failure;
5424 }
5425 if (op_array->last_var > 100) {
5426 /* To many CVs to unroll */
5427 if (!zend_jit_free_cvs(&dasm_state)) {
5428 goto jit_failure;
5429 }
5430 left_frame = 1;
5431 }
5432 if (!left_frame) {
5433 for (j = 0 ; j < op_array->last_var; j++) {
5434 uint32_t info;
5435 zend_uchar type;
5436
5437 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5438 type = STACK_TYPE(stack, j);
5439 info = zend_jit_trace_type_to_info_ex(type, info);
5440 if (opline->op1_type == IS_CV
5441 && EX_VAR_TO_NUM(opline->op1.var) == j
5442 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5443 if (JIT_G(current_frame)
5444 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5445 continue;
5446 } else {
5447 info |= MAY_BE_NULL;
5448 }
5449 }
5450 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5451 if (!left_frame) {
5452 left_frame = 1;
5453 if (!zend_jit_leave_frame(&dasm_state)) {
5454 goto jit_failure;
5455 }
5456 }
5457 if (!zend_jit_free_cv(&dasm_state, info, j)) {
5458 goto jit_failure;
5459 }
5460 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5461 if (info & MAY_BE_RC1) {
5462 may_throw = 1;
5463 }
5464 }
5465 }
5466 }
5467 }
5468 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
5469 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5470 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5471 goto jit_failure;
5472 }
5473 }
5474 goto done;
5475 case ZEND_BOOL:
5476 case ZEND_BOOL_NOT:
5477 op1_info = OP1_INFO();
5478 CHECK_OP1_TRACE_TYPE();
5479 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5480 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5481 -1, -1,
5482 zend_may_throw(opline, ssa_op, op_array, ssa),
5483 opline->opcode, NULL)) {
5484 goto jit_failure;
5485 }
5486 goto done;
5487 case ZEND_JMPZ:
5488 case ZEND_JMPNZ:
5489 case ZEND_JMPZ_EX:
5490 case ZEND_JMPNZ_EX:
5491 op1_info = OP1_INFO();
5492 CHECK_OP1_TRACE_TYPE();
5493 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5494 const zend_op *exit_opline = NULL;
5495 uint32_t exit_point;
5496
5497 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5498 /* taken branch */
5499 if (opline->opcode == ZEND_JMPNZ_EX) {
5500 smart_branch_opcode = ZEND_JMPZ_EX;
5501 } else if (opline->opcode == ZEND_JMPZ_EX) {
5502 smart_branch_opcode = ZEND_JMPNZ_EX;
5503 } else if (opline->opcode == ZEND_JMPNZ) {
5504 smart_branch_opcode = ZEND_JMPZ;
5505 } else {
5506 smart_branch_opcode = ZEND_JMPNZ;
5507 }
5508 exit_opline = opline + 1;
5509 } else if ((p+1)->opline == opline + 1) {
5510 /* not taken branch */
5511 smart_branch_opcode = opline->opcode;
5512 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5513 } else {
5514 ZEND_UNREACHABLE();
5515 }
5516 if (ra) {
5517 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5518 }
5519 if (!(op1_info & MAY_BE_GUARD)
5520 && has_concrete_type(op1_info)
5521 && concrete_type(op1_info) <= IS_TRUE) {
5522 /* unconditional branch */
5523 exit_addr = NULL;
5524 } else if (opline->result_type == IS_TMP_VAR) {
5525 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5526 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5527
5528 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5529 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5530 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5531 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5532 if (!exit_addr) {
5533 goto jit_failure;
5534 }
5535 } else {
5536 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5537 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5538 if (!exit_addr) {
5539 goto jit_failure;
5540 }
5541 }
5542 } else {
5543 ZEND_UNREACHABLE();
5544 }
5545 if (opline->result_type == IS_UNDEF) {
5546 res_addr = 0;
5547 } else {
5548 res_addr = RES_REG_ADDR();
5549 }
5550 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5551 op1_info, OP1_REG_ADDR(), res_addr,
5552 -1, -1,
5553 zend_may_throw(opline, ssa_op, op_array, ssa),
5554 smart_branch_opcode, exit_addr)) {
5555 goto jit_failure;
5556 }
5557 goto done;
5558 case ZEND_ISSET_ISEMPTY_CV:
5559 if ((opline->extended_value & ZEND_ISEMPTY)) {
5560 // TODO: support for empty() ???
5561 break;
5562 }
5563 op1_info = OP1_INFO();
5564 op1_addr = OP1_REG_ADDR();
5565 if (orig_op1_type != IS_UNKNOWN
5566 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5567 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5568 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5569 goto jit_failure;
5570 }
5571 if (opline->op1_type == IS_CV
5572 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5573 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5574 }
5575 } else {
5576 CHECK_OP1_TRACE_TYPE();
5577 }
5578 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5579 bool exit_if_true = 0;
5580 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5581 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5582
5583 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5584 if (!exit_addr) {
5585 goto jit_failure;
5586 }
5587 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5588 } else {
5589 smart_branch_opcode = 0;
5590 exit_addr = NULL;
5591 }
5592 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
5593 op1_info, op1_addr,
5594 smart_branch_opcode, -1, -1, exit_addr)) {
5595 goto jit_failure;
5596 }
5597 goto done;
5598 case ZEND_IN_ARRAY:
5599 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5600 break;
5601 }
5602 op1_info = OP1_INFO();
5603 op1_addr = OP1_REG_ADDR();
5604 CHECK_OP1_TRACE_TYPE();
5605 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5606 break;
5607 }
5608 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5609 bool exit_if_true = 0;
5610 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5611 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5612
5613 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5614 if (!exit_addr) {
5615 goto jit_failure;
5616 }
5617 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5618 } else {
5619 smart_branch_opcode = 0;
5620 exit_addr = NULL;
5621 }
5622 if (!zend_jit_in_array(&dasm_state, opline,
5623 op1_info, op1_addr,
5624 smart_branch_opcode, -1, -1, exit_addr)) {
5625 goto jit_failure;
5626 }
5627 goto done;
5628 case ZEND_FETCH_DIM_FUNC_ARG:
5629 if (!JIT_G(current_frame)
5630 || !JIT_G(current_frame)->call
5631 || !JIT_G(current_frame)->call->func
5632 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5633 break;
5634 }
5635 ZEND_FALLTHROUGH;
5636 case ZEND_FETCH_DIM_R:
5637 case ZEND_FETCH_DIM_IS:
5638 case ZEND_FETCH_LIST_R:
5639 op1_info = OP1_INFO();
5640 op1_addr = OP1_REG_ADDR();
5641 if (orig_op1_type != IS_UNKNOWN
5642 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5643 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5644 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5645 goto jit_failure;
5646 }
5647 if (opline->op1_type == IS_CV
5648 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5649 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5650 if (ssa_op->op1_def >= 0) {
5651 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5652 }
5653 }
5654 } else {
5655 CHECK_OP1_TRACE_TYPE();
5656 }
5657 op2_info = OP2_INFO();
5658 CHECK_OP2_TRACE_TYPE();
5659 res_info = RES_INFO();
5660 avoid_refcounting =
5661 ssa_op->op1_use >= 0 &&
5662 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5663 if (op1_info & MAY_BE_PACKED_GUARD) {
5664 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5665 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5666 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5667 && MAY_BE_PACKED(op1_info)
5668 && MAY_BE_HASH(op1_info)
5669 && orig_op1_type != IS_UNKNOWN) {
5670 op1_info |= MAY_BE_PACKED_GUARD;
5671 if (orig_op1_type & IS_TRACE_PACKED) {
5672 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5673 if (op1_type != IS_UNKNOWN) {
5674 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5675 }
5676 } else {
5677 op1_info &= ~MAY_BE_ARRAY_PACKED;
5678 if (op1_type != IS_UNKNOWN) {
5679 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5680 }
5681 }
5682 }
5683 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
5684 op1_info, op1_addr, avoid_refcounting,
5685 op2_info, res_info, RES_REG_ADDR(), val_type)) {
5686 goto jit_failure;
5687 }
5688 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5689 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5690 }
5691 goto done;
5692 case ZEND_FETCH_DIM_W:
5693 case ZEND_FETCH_DIM_RW:
5694 // case ZEND_FETCH_DIM_UNSET:
5695 case ZEND_FETCH_LIST_W:
5696 if (opline->op1_type != IS_CV
5697 && (orig_op1_type == IS_UNKNOWN
5698 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5699 break;
5700 }
5701 op1_info = OP1_INFO();
5702 op1_addr = OP1_REG_ADDR();
5703 if (opline->op1_type == IS_VAR) {
5704 if (orig_op1_type != IS_UNKNOWN
5705 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5706 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5707 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5708 goto jit_failure;
5709 }
5710 } else {
5711 break;
5712 }
5713 }
5714 if (orig_op1_type != IS_UNKNOWN
5715 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5716 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5717 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5718 goto jit_failure;
5719 }
5720 if (opline->op1_type == IS_CV
5721 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5722 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5723 }
5724 } else {
5725 CHECK_OP1_TRACE_TYPE();
5726 }
5727 op2_info = OP2_INFO();
5728 CHECK_OP2_TRACE_TYPE();
5729 op1_def_info = OP1_DEF_INFO();
5730 if (!zend_jit_fetch_dim(&dasm_state, opline,
5731 op1_info, op1_addr, op2_info, RES_REG_ADDR(), val_type)) {
5732 goto jit_failure;
5733 }
5734 if (ssa_op->result_def >= 0
5735 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5736 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5737 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5738 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5739 }
5740 goto done;
5741 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5742 if ((opline->extended_value & ZEND_ISEMPTY)) {
5743 // TODO: support for empty() ???
5744 break;
5745 }
5746 op1_info = OP1_INFO();
5747 op1_addr = OP1_REG_ADDR();
5748 if (orig_op1_type != IS_UNKNOWN
5749 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5750 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5751 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5752 goto jit_failure;
5753 }
5754 if (opline->op1_type == IS_CV
5755 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5756 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5757 }
5758 } else {
5759 CHECK_OP1_TRACE_TYPE();
5760 }
5761 op2_info = OP2_INFO();
5762 CHECK_OP2_TRACE_TYPE();
5763 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5764 bool exit_if_true = 0;
5765 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5766 uint32_t exit_point;
5767
5768 if (ra) {
5769 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5770 }
5771 if (ssa_op->op1_use >= 0
5772 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5773 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5774 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5775 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5776
5777 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5778 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5779 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5780 } else {
5781 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5782 }
5783 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5784 if (!exit_addr) {
5785 goto jit_failure;
5786 }
5787 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5788 } else {
5789 smart_branch_opcode = 0;
5790 exit_addr = NULL;
5791 }
5792 avoid_refcounting =
5793 ssa_op->op1_use >= 0 &&
5794 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5795 if (op1_info & MAY_BE_PACKED_GUARD) {
5796 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5797 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5798 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5799 && MAY_BE_PACKED(op1_info)
5800 && MAY_BE_HASH(op1_info)
5801 && orig_op1_type != IS_UNKNOWN) {
5802 op1_info |= MAY_BE_PACKED_GUARD;
5803 if (orig_op1_type & IS_TRACE_PACKED) {
5804 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5805 } else {
5806 op1_info &= ~MAY_BE_ARRAY_PACKED;
5807 }
5808 }
5809 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
5810 op1_info, op1_addr, avoid_refcounting,
5811 op2_info, val_type,
5812 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5813 smart_branch_opcode, -1, -1,
5814 exit_addr)) {
5815 goto jit_failure;
5816 }
5817 goto done;
5818 case ZEND_FETCH_OBJ_FUNC_ARG:
5819 if (!JIT_G(current_frame)
5820 || !JIT_G(current_frame)->call
5821 || !JIT_G(current_frame)->call->func
5822 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5823 break;
5824 }
5825 ZEND_FALLTHROUGH;
5826 case ZEND_FETCH_OBJ_R:
5827 case ZEND_FETCH_OBJ_IS:
5828 case ZEND_FETCH_OBJ_W:
5829 on_this = delayed_fetch_this = 0;
5830 avoid_refcounting = 0;
5831 if (opline->op2_type != IS_CONST
5832 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5833 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5834 break;
5835 }
5836 ce = NULL;
5837 ce_is_instanceof = 0;
5838 op1_indirect = 0;
5839 if (opline->op1_type == IS_UNUSED) {
5840 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
5841 ce = op_array->scope;
5842 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
5843 op1_addr = 0;
5844 on_this = 1;
5845 } else {
5846 op1_info = OP1_INFO();
5847 if (!(op1_info & MAY_BE_OBJECT)) {
5848 break;
5849 }
5850 op1_addr = OP1_REG_ADDR();
5851 if (opline->op1_type == IS_VAR
5852 && opline->opcode == ZEND_FETCH_OBJ_W) {
5853 if (orig_op1_type != IS_UNKNOWN
5854 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5855 op1_indirect = 1;
5856 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5857 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5858 goto jit_failure;
5859 }
5860 }
5861 }
5862 if (orig_op1_type != IS_UNKNOWN
5863 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5864 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5865 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5866 goto jit_failure;
5867 }
5868 if (opline->op1_type == IS_CV
5869 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5870 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
5871 }
5872 } else {
5873 CHECK_OP1_TRACE_TYPE();
5874 }
5875 if (!(op1_info & MAY_BE_OBJECT)) {
5876 break;
5877 }
5878 if (ssa->var_info && ssa->ops) {
5879 if (ssa_op->op1_use >= 0) {
5880 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5881 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5882 ce = op1_ssa->ce;
5883 ce_is_instanceof = op1_ssa->is_instanceof;
5884 }
5885 }
5886 }
5887 if (ssa_op->op1_use >= 0) {
5888 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
5889 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5890 }
5891 if (delayed_fetch_this) {
5892 on_this = 1;
5893 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
5894 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
5895 } else if (op_array_ssa->ops
5896 && op_array_ssa->vars
5897 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
5898 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
5899 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
5900 }
5901 }
5902 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
5903 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
5904 on_this, delayed_fetch_this, avoid_refcounting, op1_ce, val_type,
5905 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
5906 goto jit_failure;
5907 }
5908 goto done;
5909 case ZEND_BIND_GLOBAL:
5910 orig_opline = opline;
5911 orig_ssa_op = ssa_op;
5912 while (1) {
5913 if (!ssa->ops || !ssa->var_info) {
5914 op1_info = MAY_BE_ANY|MAY_BE_REF;
5915 } else {
5916 op1_info = OP1_INFO();
5917 }
5918 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5919 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5920 }
5921 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
5922 goto jit_failure;
5923 }
5924 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
5925 opline++;
5926 ssa_op++;
5927 } else {
5928 break;
5929 }
5930 }
5931 opline = orig_opline;
5932 ssa_op = orig_ssa_op;
5933 goto done;
5934 case ZEND_RECV:
5935 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
5936 goto jit_failure;
5937 }
5938 goto done;
5939 case ZEND_RECV_INIT:
5940 orig_opline = opline;
5941 orig_ssa_op = ssa_op;
5942 while (1) {
5943 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
5944 (opline + 1)->opcode != ZEND_RECV_INIT,
5945 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5946 goto jit_failure;
5947 }
5948 if ((opline+1)->opcode == ZEND_RECV_INIT) {
5949 opline++;
5950 ssa_op++;
5951 } else {
5952 break;
5953 }
5954 }
5955 opline = orig_opline;
5956 ssa_op = orig_ssa_op;
5957 goto done;
5958 case ZEND_FREE:
5959 case ZEND_FE_FREE:
5960 op1_info = OP1_INFO();
5961 if (!zend_jit_free(&dasm_state, opline, op1_info,
5962 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5963 goto jit_failure;
5964 }
5965 goto done;
5966 case ZEND_ECHO:
5967 op1_info = OP1_INFO();
5968 CHECK_OP1_TRACE_TYPE();
5969 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5970 break;
5971 }
5972 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
5973 goto jit_failure;
5974 }
5975 goto done;
5976 case ZEND_STRLEN:
5977 op1_info = OP1_INFO();
5978 op1_addr = OP1_REG_ADDR();
5979 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
5980 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5981 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5982 goto jit_failure;
5983 }
5984 if (opline->op1_type == IS_CV
5985 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5986 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5987 }
5988 } else {
5989 CHECK_OP1_TRACE_TYPE();
5990 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5991 break;
5992 }
5993 }
5994 if (!zend_jit_strlen(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR())) {
5995 goto jit_failure;
5996 }
5997 goto done;
5998 case ZEND_COUNT:
5999 op1_info = OP1_INFO();
6000 op1_addr = OP1_REG_ADDR();
6001 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
6002 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6003 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6004 goto jit_failure;
6005 }
6006 if (opline->op1_type == IS_CV
6007 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6008 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6009 }
6010 } else {
6011 CHECK_OP1_TRACE_TYPE();
6012 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6013 break;
6014 }
6015 }
6016 if (!zend_jit_count(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6017 goto jit_failure;
6018 }
6019 goto done;
6020 case ZEND_FETCH_THIS:
6021 delayed_fetch_this = 0;
6022 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6023 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6024 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6025 delayed_fetch_this = 1;
6026 }
6027 }
6028 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) {
6029 goto jit_failure;
6030 }
6031 goto done;
6032 case ZEND_SWITCH_LONG:
6033 case ZEND_SWITCH_STRING:
6034 case ZEND_MATCH:
6035 if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6036 goto jit_failure;
6037 }
6038 goto done;
6039 case ZEND_VERIFY_RETURN_TYPE:
6040 if (opline->op1_type == IS_UNUSED) {
6041 /* Always throws */
6042 break;
6043 }
6044 if (opline->op1_type == IS_CONST) {
6045 /* TODO Different instruction format, has return value */
6046 break;
6047 }
6048 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6049 /* Not worth bothering with */
6050 break;
6051 }
6052 op1_info = OP1_INFO();
6053 CHECK_OP1_TRACE_TYPE();
6054 if (op1_info & MAY_BE_REF) {
6055 /* TODO May need reference unwrapping. */
6056 break;
6057 }
6058 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, op1_info)) {
6059 goto jit_failure;
6060 }
6061 goto done;
6062 case ZEND_FE_RESET_R:
6063 op1_info = OP1_INFO();
6064 CHECK_OP1_TRACE_TYPE();
6065 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6066 break;
6067 }
6068 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
6069 goto jit_failure;
6070 }
6071 goto done;
6072 case ZEND_FE_FETCH_R:
6073 op1_info = OP1_INFO();
6074 CHECK_OP1_TRACE_TYPE();
6075 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6076 break;
6077 }
6078 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6079 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6080 uint32_t exit_point;
6081
6082 if ((p+1)->opline == exit_opline) {
6083 /* taken branch (exit from loop) */
6084 exit_opline = opline;
6085 smart_branch_opcode = ZEND_NOP;
6086 } else if ((p+1)->opline == opline + 1) {
6087 /* not taken branch (loop) */
6088 smart_branch_opcode = ZEND_JMP;
6089 } else {
6090 ZEND_UNREACHABLE();
6091 }
6092 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6093 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6094 if (!exit_addr) {
6095 goto jit_failure;
6096 }
6097 } else {
6098 ZEND_UNREACHABLE();
6099 }
6100 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
6101 -1, smart_branch_opcode, exit_addr)) {
6102 goto jit_failure;
6103 }
6104 goto done;
6105 case ZEND_FETCH_CONSTANT:
6106 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6107 goto jit_failure;
6108 }
6109 goto done;
6110 case ZEND_INIT_METHOD_CALL:
6111 if (opline->op2_type != IS_CONST
6112 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6113 break;
6114 }
6115 on_this = delayed_fetch_this = 0;
6116 ce = NULL;
6117 ce_is_instanceof = 0;
6118 if (opline->op1_type == IS_UNUSED) {
6119 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6120 ce = op_array->scope;
6121 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
6122 op1_addr = 0;
6123 on_this = 1;
6124 } else {
6125 op1_info = OP1_INFO();
6126 op1_addr = OP1_REG_ADDR();
6127 if (polymorphic_side_trace) {
6128 op1_info = MAY_BE_OBJECT;
6129 op1_addr = 0;
6130 } else if (orig_op1_type != IS_UNKNOWN
6131 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6132 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6133 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6134 goto jit_failure;
6135 }
6136 if (opline->op1_type == IS_CV
6137 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6138 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6139 }
6140 } else {
6141 CHECK_OP1_TRACE_TYPE();
6142 }
6143 if (ssa->var_info && ssa->ops) {
6144 if (ssa_op->op1_use >= 0) {
6145 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6146 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6147 ce = op1_ssa->ce;
6148 ce_is_instanceof = op1_ssa->is_instanceof;
6149 }
6150 }
6151 }
6152 if (ssa_op->op1_use >= 0) {
6153 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6154 }
6155 if (delayed_fetch_this) {
6156 on_this = 1;
6157 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6158 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6159 } else if (op_array_ssa->ops
6160 && op_array_ssa->vars
6161 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6162 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6163 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6164 }
6165 }
6166 frame_flags = TRACE_FRAME_MASK_NESTED;
6167 if (!zend_jit_init_method_call(&dasm_state, opline,
6168 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6169 op_array, ssa, ssa_op, frame->call_level,
6170 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6171 p + 1, peek_checked_stack - checked_stack, polymorphic_side_trace)) {
6172 goto jit_failure;
6173 }
6174 goto done;
6175 case ZEND_INIT_DYNAMIC_CALL:
6176 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6177 break;
6178 }
6179 op2_info = OP2_INFO();
6180 CHECK_OP2_TRACE_TYPE();
6181 frame_flags = TRACE_FRAME_MASK_NESTED;
6182 if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6183 goto jit_failure;
6184 }
6185 goto done;
6186 case ZEND_SEND_ARRAY:
6187 case ZEND_SEND_UNPACK:
6188 if (JIT_G(current_frame)
6189 && JIT_G(current_frame)->call) {
6190 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6191 }
6192 break;
6193 case ZEND_ROPE_INIT:
6194 case ZEND_ROPE_ADD:
6195 case ZEND_ROPE_END:
6196 op2_info = OP2_INFO();
6197 CHECK_OP2_TRACE_TYPE();
6198 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6199 break;
6200 }
6201 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
6202 goto jit_failure;
6203 }
6204 goto done;
6205 default:
6206 break;
6207 }
6208 }
6209
6210 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6211 gen_handler = 1;
6212 op1_info = OP1_INFO();
6213 op2_info = OP2_INFO();
6214 if (op1_info & MAY_BE_GUARD) {
6215 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6216 }
6217 if (op2_info & MAY_BE_GUARD) {
6218 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6219 }
6220 if (!zend_jit_trace_handler(&dasm_state, op_array, opline,
6221 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6222 goto jit_failure;
6223 }
6224 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
6225 if (opline->opcode == ZEND_NEW && opline->result_type != IS_UNUSED) {
6226 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6227 }
6228 if (zend_jit_may_be_polymorphic_call(opline) ||
6229 zend_jit_may_be_modified((p+1)->func, op_array)) {
6230 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
6231 goto jit_failure;
6232 }
6233 }
6234 }
6235 }
6236
6237 done:
6238 polymorphic_side_trace = 0;
6239 switch (opline->opcode) {
6240 case ZEND_DO_FCALL:
6241 case ZEND_DO_ICALL:
6242 case ZEND_DO_UCALL:
6243 case ZEND_DO_FCALL_BY_NAME:
6244 case ZEND_CALLABLE_CONVERT:
6245 frame->call_level--;
6246 }
6247
6248 if (ra) {
6249 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
6250 }
6251
6252 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6253 && STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var)) > ZREG_NUM) {
6254 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6255 }
6256
6257 if (opline->opcode == ZEND_ROPE_INIT) {
6258 /* clear stack slots used by rope */
6259 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6260 uint32_t count =
6261 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6262
6263 do {
6264 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6265 var++;
6266 count--;
6267 } while (count);
6268 }
6269
6270 if (ssa_op) {
6271 zend_ssa_range tmp;
6272
6273 /* Keep information about known types on abstract stack */
6274 if (ssa_op->result_def >= 0) {
6275 zend_uchar type = IS_UNKNOWN;
6276
6277 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6278 || send_result) {
6279 /* we didn't set result variable */
6280 type = IS_UNKNOWN;
6281 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6282 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6283 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6284 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6285 if (opline->op1_type != IS_CONST) {
6286 /* copy */
6287 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6288 }
6289 } else if (opline->opcode == ZEND_ASSIGN) {
6290 if (opline->op2_type != IS_CONST
6291 && ssa_op->op1_use >= 0
6292 /* assignment to typed reference may cause conversion */
6293 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6294 /* copy */
6295 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6296 }
6297 } else if (opline->opcode == ZEND_POST_INC
6298 || opline->opcode == ZEND_POST_DEC) {
6299 /* copy */
6300 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6301 }
6302 if (opline->opcode == ZEND_JMP_SET
6303 || opline->opcode == ZEND_COALESCE
6304 || opline->opcode == ZEND_JMP_NULL) {
6305 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6306 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6307 } else if ((p+1)->opline != (opline + 1)) {
6308 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6309 }
6310 } else {
6311 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6312 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->result_def]));
6313 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6314 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6315 }
6316 if (type != IS_UNKNOWN) {
6317 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6318 if (opline->opcode == ZEND_FETCH_THIS
6319 && delayed_fetch_this) {
6320 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_THIS);
6321 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6322 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_TRY_ADDREF);
6323 } else if (ra && ra[ssa_op->result_def]) {
6324 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6325 ra[ssa_op->result_def]->flags & ZREG_STORE);
6326 }
6327 }
6328 }
6329
6330 if (type == IS_LONG
6331 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->result_def, &tmp)) {
6332 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6333 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6334 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6335 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6336 ssa->var_info[ssa_op->result_def].has_range = 1;
6337 }
6338 }
6339 if (ssa_op->op1_def >= 0
6340 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6341 || opline->result_type != IS_CV
6342 || opline->result.var != opline->op1.var)) {
6343 zend_uchar type = IS_UNKNOWN;
6344
6345 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6346 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6347 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6348 } else if (opline->opcode == ZEND_ASSIGN) {
6349 if (!(OP1_INFO() & MAY_BE_REF)
6350 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6351 if (opline->op2_type != IS_CONST) {
6352 /* copy */
6353 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6354 }
6355 }
6356 } else if (opline->opcode == ZEND_SEND_VAR
6357 || opline->opcode == ZEND_CAST
6358 || opline->opcode == ZEND_QM_ASSIGN
6359 || opline->opcode == ZEND_JMP_SET
6360 || opline->opcode == ZEND_COALESCE
6361 || opline->opcode == ZEND_JMP_NULL
6362 || opline->opcode == ZEND_FE_RESET_R) {
6363 /* keep old value */
6364 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6365 }
6366 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6367 (gen_handler || type == IS_UNKNOWN || !ra ||
6368 (!ra[ssa_op->op1_def] &&
6369 !(ssa->vars[ssa_op->op1_def].no_val &&
6370 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6371 (opline->opcode == ZEND_QM_ASSIGN ||
6372 opline->opcode == ZEND_SEND_VAR ||
6373 opline->opcode == ZEND_SEND_VAR_EX ||
6374 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6375 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6376 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6377 if (type != IS_UNKNOWN) {
6378 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6379 if (ra && ra[ssa_op->op1_def]) {
6380 uint8_t flags = ra[ssa_op->op1_def]->flags & ZREG_STORE;
6381
6382 if (ssa_op->op1_use >= 0) {
6383 if (opline->opcode == ZEND_SEND_VAR
6384 || opline->opcode == ZEND_CAST
6385 || opline->opcode == ZEND_QM_ASSIGN
6386 || opline->opcode == ZEND_JMP_SET
6387 || opline->opcode == ZEND_COALESCE
6388 || opline->opcode == ZEND_JMP_NULL
6389 || opline->opcode == ZEND_FE_RESET_R) {
6390 if (!ra[ssa_op->op1_use]) {
6391 flags |= ZREG_LOAD;
6392 }
6393 }
6394 }
6395 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, flags);
6396 }
6397 }
6398 if (type == IS_LONG
6399 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op1_def, &tmp)) {
6400 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6401 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6402 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6403 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6404 ssa->var_info[ssa_op->op1_def].has_range = 1;
6405 }
6406 }
6407 if (ssa_op->op2_def >= 0
6408 && (opline->opcode != ZEND_ASSIGN
6409 || opline->op1_type != IS_CV
6410 || opline->op1.var != opline->op2.var)) {
6411 zend_uchar type = IS_UNKNOWN;
6412
6413 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6414 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6415 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6416 } else if (opline->opcode == ZEND_ASSIGN) {
6417 /* keep old value */
6418 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6419 }
6420 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6421 (gen_handler || type == IS_UNKNOWN || !ra ||
6422 (!ra[ssa_op->op2_def] &&
6423 !(ssa->vars[ssa_op->op2_def].no_val &&
6424 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6425 opline->opcode == ZEND_ASSIGN))));
6426 if (type != IS_UNKNOWN) {
6427 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6428 if (ra && ra[ssa_op->op2_def]) {
6429 uint8_t flags = ra[ssa_op->op2_def]->flags & ZREG_STORE;
6430
6431 if (ssa_op->op2_use >= 0) {
6432 if (opline->opcode == ZEND_ASSIGN) {
6433 if (!ra[ssa_op->op2_use]
6434 || ra[ssa_op->op2_use]->reg != ra[ssa_op->op2_def]->reg) {
6435 flags |= ZREG_LOAD;
6436 }
6437 }
6438 }
6439 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def]->reg, flags);
6440 }
6441 }
6442 if (type == IS_LONG
6443 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op2_def, &tmp)) {
6444 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6445 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6446 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6447 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6448 ssa->var_info[ssa_op->op2_def].has_range = 1;
6449 }
6450 }
6451
6452 switch (opline->opcode) {
6453 case ZEND_ASSIGN_DIM:
6454 case ZEND_ASSIGN_OBJ:
6455 case ZEND_ASSIGN_STATIC_PROP:
6456 case ZEND_ASSIGN_DIM_OP:
6457 case ZEND_ASSIGN_OBJ_OP:
6458 case ZEND_ASSIGN_STATIC_PROP_OP:
6459 case ZEND_ASSIGN_OBJ_REF:
6460 case ZEND_ASSIGN_STATIC_PROP_REF:
6461 /* OP_DATA */
6462 ssa_op++;
6463 opline++;
6464 if (ssa_op->op1_def >= 0) {
6465 zend_uchar type = IS_UNKNOWN;
6466
6467 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6468 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6469 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6470 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6471 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6472 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6473 /* keep old value */
6474 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6475 }
6476 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6477 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->op1_def]));
6478 if (type != IS_UNKNOWN) {
6479 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6480 if (ra && ra[ssa_op->op1_def]) {
6481 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6482 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6483 }
6484 }
6485 if (type == IS_LONG
6486 && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op1_def, &tmp)) {
6487 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6488 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6489 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6490 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6491 ssa->var_info[ssa_op->op1_def].has_range = 1;
6492 }
6493 }
6494 ssa_op++;
6495 break;
6496 case ZEND_RECV_INIT:
6497 ssa_op++;
6498 opline++;
6499 while (opline->opcode == ZEND_RECV_INIT) {
6500 if (ssa_op->result_def >= 0) {
6501 zend_uchar type = IS_UNKNOWN;
6502
6503 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6504 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6505 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6506 }
6507 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6508 (gen_handler || !ra || !ra[ssa_op->result_def]));
6509 if (ra && ra[ssa_op->result_def]) {
6510 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6511 ra[ssa_op->result_def]->flags & ZREG_STORE);
6512 }
6513 }
6514 ssa_op++;
6515 opline++;
6516 }
6517 break;
6518 case ZEND_BIND_GLOBAL:
6519 ssa_op++;
6520 opline++;
6521 while (opline->opcode == ZEND_BIND_GLOBAL) {
6522 if (ssa_op->op1_def >= 0) {
6523 zend_uchar type = IS_UNKNOWN;
6524
6525 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6526 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6527 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6528 }
6529 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6530 (gen_handler || !ra || !ra[ssa_op->op1_def]));
6531 if (ra && ra[ssa_op->op1_def]) {
6532 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6533 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6534 }
6535 }
6536 ssa_op++;
6537 opline++;
6538 }
6539 break;
6540 default:
6541 ssa_op += zend_jit_trace_op_len(opline);
6542 break;
6543 }
6544
6545 if (send_result) {
6546 ssa_op++;
6547 p++;
6548 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6549 p++;
6550 }
6551 send_result = 0;
6552 }
6553 }
6554 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6555 call = frame->call;
6556 assert(call && &call->func->op_array == p->op_array);
6557
6558 if (opline->opcode == ZEND_DO_UCALL
6559 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6560 || opline->opcode == ZEND_DO_FCALL) {
6561
6562 frame->call_opline = opline;
6563
6564 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6565 if (opline > op_array->opcodes) {
6566 const zend_op *prev_opline = opline - 1;
6567
6568 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6569 prev_opline--;
6570 }
6571 JIT_G(current_frame) = call;
6572 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6573 || prev_opline->opcode == ZEND_SEND_UNPACK
6574 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6575 && p->op_array->num_args
6576 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6577 && ((p+1)->op == ZEND_JIT_TRACE_VM
6578 || (p+1)->op == ZEND_JIT_TRACE_END)
6579 && (TRACE_FRAME_NUM_ARGS(call) < 0
6580 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6581 && !zend_jit_trace_opline_guard(&dasm_state, (p+1)->opline)) {
6582 goto jit_failure;
6583 }
6584 JIT_G(current_frame) = frame;
6585 }
6586 }
6587
6588 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6589 p++;
6590 break;
6591 }
6592 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6593 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6594 TRACE_FRAME_SET_THIS_CHECKED(call);
6595 }
6596 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6597 TRACE_FRAME_SET_THIS_CHECKED(call);
6598 }
6599 op_array = (zend_op_array*)p->op_array;
6600 jit_extension =
6601 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6602 op_array_ssa = &jit_extension->func_info.ssa;
6603 frame->call = call->prev;
6604 call->prev = frame;
6605 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6606 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6607 } else {
6608 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6609 }
6610 JIT_G(current_frame) = frame = call;
6611 stack = frame->stack;
6612 if (ra) {
6613 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6614
6615 for (i = 0; i < op_array->last_var; i++,j++) {
6616 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6617 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6618 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6619 goto jit_failure;
6620 }
6621 }
6622 }
6623 }
6624 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6625 op_array = (zend_op_array*)p->op_array;
6626 jit_extension =
6627 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6628 op_array_ssa = &jit_extension->func_info.ssa;
6629 top = frame;
6630 if (frame->prev) {
6631 checked_stack = frame->old_checked_stack;
6632 peek_checked_stack = frame->old_peek_checked_stack;
6633 frame = frame->prev;
6634 stack = frame->stack;
6635 ZEND_ASSERT(&frame->func->op_array == op_array);
6636 } else {
6637 frame = zend_jit_trace_ret_frame(frame, op_array);
6638 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6639 frame->used_stack = checked_stack = peek_checked_stack = 0;
6640 stack = frame->stack;
6641 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6642 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6643
6644 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6645 /* Initialize abstract stack using SSA */
6646 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6647 && has_concrete_type(ssa->var_info[j].type)) {
6648 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6649 } else {
6650 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6651 }
6652 }
6653 if (ra) {
6654 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6655 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6656 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6657 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6658 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6659 goto jit_failure;
6660 }
6661 }
6662 }
6663 }
6664 } else {
6665 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6666 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6667 }
6668 }
6669 opline = NULL;
6670 }
6671 JIT_G(current_frame) = frame;
6672 if (res_type != IS_UNKNOWN
6673 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6674 const zend_op *opline = (p+1)->opline - 1;
6675 if (opline->result_type != IS_UNUSED) {
6676 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6677 }
6678 }
6679 res_type = IS_UNKNOWN;
6680 } else if (p->op == ZEND_JIT_TRACE_END) {
6681 break;
6682 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6683 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6684 int num_args = -1;
6685
6686 if (init_opline
6687 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6688 num_args = init_opline->extended_value;
6689 }
6690
6691 call = top;
6692 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6693 call->prev = frame->call;
6694 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6695 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6696 if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
6697 TRACE_FRAME_SET_CLOSURE_CALL(call);
6698 }
6699 }
6700 if (init_opline) {
6701 if (init_opline->opcode != ZEND_NEW
6702 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6703 || init_opline->op1_type == IS_UNDEF
6704 || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6705 && ssa_op
6706 && (ssa_op-1)->op1_use >=0
6707 && ssa->var_info[(ssa_op-1)->op1_use].delayed_fetch_this))
6708 && (init_opline->opcode != ZEND_INIT_USER_CALL
6709 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6710 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6711 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6712 ) {
6713 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6714 } else if (init_opline->opcode == ZEND_NEW
6715 || (init_opline->opcode == ZEND_INIT_METHOD_CALL
6716 && init_opline->op1_type != IS_UNDEF
6717 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6718 && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) {
6719 TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call);
6720 }
6721 }
6722 frame->call = call;
6723 top = zend_jit_trace_call_frame(top, p->op_array);
6724 if (p->func) {
6725 if (p->func->type == ZEND_USER_FUNCTION) {
6726 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6727 zend_jit_op_array_trace_extension *jit_extension =
6728 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6729
6730 i = 0;
6731 while (i < p->op_array->num_args) {
6732 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6733 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6734 i++;
6735 }
6736 while (i < p->op_array->last_var) {
6737 if (jit_extension
6738 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6739 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6740 } else {
6741 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6742 }
6743 i++;
6744 }
6745 while (i < p->op_array->last_var + p->op_array->T) {
6746 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6747 i++;
6748 }
6749 } else {
6750 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
6751 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6752 }
6753 }
6754 } else {
6755 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
6756 for (i = 0; i < p->op_array->num_args; i++) {
6757 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6758 }
6759 }
6760 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6761 int skip_guard = 0;
6762
6763 if (init_opline) {
6764 zend_call_info *call_info = jit_extension->func_info.callee_info;
6765
6766 while (call_info) {
6767 if (call_info->caller_init_opline == init_opline
6768 && !call_info->is_prototype) {
6769 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
6770 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
6771 && init_opline->op1_type != IS_CONST) {
6772 break;
6773 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
6774 break;
6775 }
6776 }
6777 skip_guard = 1;
6778 break;
6779 }
6780 call_info = call_info->next_callee;
6781 }
6782 if (!skip_guard
6783 && !zend_jit_may_be_polymorphic_call(init_opline)
6784 && !zend_jit_may_be_modified(p->func, op_array)) {
6785 skip_guard = 1;
6786 }
6787 }
6788
6789 if (!skip_guard) {
6790 if (!opline) {
6791 zend_jit_trace_rec *q = p + 1;
6792 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
6793 q++;
6794 }
6795 opline = q->opline;
6796 ZEND_ASSERT(opline != NULL);
6797 }
6798 if (!zend_jit_init_fcall_guard(&dasm_state,
6799 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
6800 goto jit_failure;
6801 }
6802 }
6803 }
6804 }
6805 call->old_checked_stack = checked_stack;
6806 call->old_peek_checked_stack = peek_checked_stack;
6807 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6808 frame->call_level++;
6809 call->used_stack = checked_stack = peek_checked_stack = 0;
6810 } else {
6811 if (p->func) {
6812 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
6813 } else {
6814 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
6815 }
6816 switch (init_opline->opcode) {
6817 case ZEND_INIT_FCALL:
6818 case ZEND_INIT_FCALL_BY_NAME:
6819 case ZEND_INIT_NS_FCALL_BY_NAME:
6820 case ZEND_INIT_METHOD_CALL:
6821 case ZEND_INIT_DYNAMIC_CALL:
6822 //case ZEND_INIT_STATIC_METHOD_CALL:
6823 //case ZEND_INIT_USER_CALL:
6824 //case ZEND_NEW:
6825 checked_stack += call->used_stack;
6826 if (checked_stack > peek_checked_stack) {
6827 peek_checked_stack = checked_stack;
6828 }
6829 break;
6830 default:
6831 checked_stack = peek_checked_stack = 0;
6832 }
6833 }
6834 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
6835 call = frame->call;
6836 if (call) {
6837 checked_stack = call->old_checked_stack;
6838 peek_checked_stack = call->old_peek_checked_stack;
6839 top = call;
6840 frame->call = call->prev;
6841 }
6842 } else {
6843 ZEND_UNREACHABLE();
6844 }
6845 }
6846
6847 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
6848
6849 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
6850
6851 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6852 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6853 }
6854
6855 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
6856 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
6857 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6858 if (ra) {
6859 zend_ssa_phi *phi = ssa->blocks[1].phis;
6860
6861 while (phi) {
6862 if (ra[phi->ssa_var]
6863 && ra[phi->sources[1]]
6864 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
6865 && (ra[phi->ssa_var]->flags & (ZREG_LOAD|ZREG_STORE)) == 0
6866 && (ra[phi->sources[1]]->flags & (ZREG_LOAD|ZREG_STORE)) == 0) {
6867 /* Store actual type to memory to avoid deoptimization mistakes */
6868 /* TODO: Alternatively, we may try to update alredy generated deoptimization info */
6869 zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var));
6870 }
6871 phi = phi->next;
6872 }
6873 }
6874 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6875 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6876 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6877 goto jit_failure;
6878 }
6879 }
6880 t->link = ZEND_JIT_TRACE_NUM;
6881 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6882 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
6883 }
6884 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
6885 const void *timeout_exit_addr = NULL;
6886
6887 t->flags |= ZEND_JIT_TRACE_LOOP;
6888
6889 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6890 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6891 || (ra
6892 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
6893 /* Deoptimize to the first instruction of the loop */
6894 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
6895
6896 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6897 if (!timeout_exit_addr) {
6898 goto jit_failure;
6899 }
6900 } else {
6901 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6902 }
6903 }
6904
6905 zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
6906 }
6907 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
6908 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
6909 if (ra
6910 && (p-1)->op != ZEND_JIT_TRACE_ENTER
6911 && (p-1)->op != ZEND_JIT_TRACE_BACK
6912 && opline->opcode != ZEND_DO_UCALL
6913 && opline->opcode != ZEND_DO_FCALL
6914 && opline->opcode != ZEND_DO_FCALL_BY_NAME
6915 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
6916 if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL,
6917 stack, op_array->last_var + op_array->T, NULL, NULL, NULL, 0)) {
6918 goto jit_failure;
6919 }
6920 for (i = 0; i < op_array->last_var; i++) {
6921 int8_t reg = STACK_REG(stack, i);
6922 uint8_t type = STACK_TYPE(stack, i);
6923
6924 if (reg == ZREG_NONE
6925 && type != IS_UNKNOWN
6926 && type != STACK_MEM_TYPE(stack, i)) {
6927 if (!zend_jit_store_var_type(&dasm_state, i, type)) {
6928 return 0;
6929 }
6930 SET_STACK_TYPE(stack, i, type, 1);
6931 }
6932 }
6933 }
6934 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
6935 const void *timeout_exit_addr = NULL;
6936
6937 t->link = zend_jit_find_trace(p->opline->handler);
6938 if (t->link == 0) {
6939 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
6940 * by zend_jit_trace_exit() in another thread after this
6941 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
6942 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
6943 * the "_counter_handler" functions, and these are not registered
6944 * tracer functions */
6945 goto jit_failure;
6946 }
6947 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6948 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6949 goto jit_failure;
6950 }
6951 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6952 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6953 }
6954 if (parent_trace
6955 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
6956 && zend_jit_traces[parent_trace].root == t->link) {
6957 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
6958 uint32_t exit_point;
6959
6960 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6961 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6962 }
6963 exit_point = zend_jit_trace_get_exit_point(zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM);
6964 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6965 if (!timeout_exit_addr) {
6966 goto jit_failure;
6967 }
6968 } else {
6969 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6970 }
6971 }
6972 zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
6973 } else {
6974 zend_jit_trace_return(&dasm_state, 0, NULL);
6975 }
6976 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
6977 zend_jit_trace_return(&dasm_state, 0, NULL);
6978 } else {
6979 // TODO: not implemented ???
6980 ZEND_ASSERT(0 && p->stop);
6981 }
6982
6983 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
6984 goto jit_failure;
6985 }
6986
6987 if (!zend_jit_trace_end(&dasm_state, t)) {
6988 goto jit_failure;
6989 }
6990
6991 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM,
6992 parent_trace ? SP_ADJ_JIT : ((zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET),
6993 parent_trace ? SP_ADJ_NONE : SP_ADJ_JIT);
6994
6995 if (handler) {
6996 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
6997 const zend_op_array *rec_op_array;
6998
6999 rec_op_array = op_array = trace_buffer->op_array;
7000 jit_extension =
7001 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7002 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7003 for (;;p++) {
7004 if (p->op == ZEND_JIT_TRACE_VM) {
7005 opline = p->opline;
7006 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7007 if (p->op_array == rec_op_array) {
7008 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7009 }
7010 op_array = p->op_array;
7011 jit_extension =
7012 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7013 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7014 op_array = p->op_array;
7015 jit_extension =
7016 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7017 } else if (p->op == ZEND_JIT_TRACE_END) {
7018 break;
7019 }
7020 }
7021 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7022 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7023 if (opline
7024 && (opline->opcode == ZEND_DO_UCALL
7025 || opline->opcode == ZEND_DO_FCALL
7026 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7027 || opline->opcode == ZEND_YIELD
7028 || opline->opcode == ZEND_YIELD_FROM
7029 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7030 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7031 }
7032 if (JIT_G(current_frame)
7033 && JIT_G(current_frame)->prev) {
7034 frame = JIT_G(current_frame)->prev;
7035 do {
7036 if (frame->call_opline) {
7037 op_array = &frame->func->op_array;
7038 jit_extension =
7039 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7040 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7041 }
7042 frame = frame->prev;
7043 } while (frame);
7044 }
7045 }
7046 }
7047
7048 jit_failure:
7049 dasm_free(&dasm_state);
7050
7051 if (name) {
7052 zend_string_release(name);
7053 }
7054
7055 jit_cleanup:
7056 /* Clean up used op_arrays */
7057 while (num_op_arrays > 0) {
7058 op_array = op_arrays[--num_op_arrays];
7059 jit_extension =
7060 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7061
7062 jit_extension->func_info.num = 0;
7063 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7064 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7065 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7066 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7067 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7068 }
7069
7070 zend_arena_release(&CG(arena), checkpoint);
7071
7072 JIT_G(current_frame) = NULL;
7073 JIT_G(current_trace) = NULL;
7074
7075 return handler;
7076 }
7077
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7078 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7079 {
7080 const void *handler = NULL;
7081 dasm_State* dasm_state = NULL;
7082 void *checkpoint;
7083 char name[32];
7084 const zend_op *opline;
7085 uint32_t stack_size;
7086 zend_jit_trace_stack *stack;
7087 bool original_handler = 0;
7088
7089 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7090 return dasm_labels[zend_lbtrace_escape];
7091 }
7092
7093 checkpoint = zend_arena_checkpoint(CG(arena));;
7094
7095 sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num);
7096
7097 dasm_init(&dasm_state, DASM_MAXSECTION);
7098 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
7099 dasm_setup(&dasm_state, dasm_actions);
7100
7101 zend_jit_align_func(&dasm_state);
7102
7103 /* Deoptimization */
7104 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7105 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
7106
7107 if (!zend_jit_trace_deoptimization(&dasm_state,
7108 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7109 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7110 stack, stack_size, NULL, NULL, NULL, 0)) {
7111 goto jit_failure;
7112 }
7113
7114 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7115 if (opline) {
7116 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7117 /* prevent endless loop */
7118 original_handler = 1;
7119 }
7120 zend_jit_set_ip_ex(&dasm_state, opline, original_handler);
7121 }
7122
7123 zend_jit_trace_return(&dasm_state, original_handler, opline);
7124
7125 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM, SP_ADJ_JIT, SP_ADJ_NONE);
7126
7127 jit_failure:
7128 dasm_free(&dasm_state);
7129 zend_arena_release(&CG(arena), checkpoint);
7130 return handler;
7131 }
7132
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7133 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7134 {
7135 zend_jit_trace_stop ret;
7136 const void *handler;
7137 uint8_t orig_trigger;
7138 zend_jit_trace_info *t = NULL;
7139 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7140 bool do_bailout = 0;
7141
7142 zend_shared_alloc_lock();
7143
7144 /* Checks under lock */
7145 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7146 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7147 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7148 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7149 } else {
7150 zend_try {
7151 SHM_UNPROTECT();
7152 zend_jit_unprotect();
7153
7154 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7155
7156 t->id = ZEND_JIT_TRACE_NUM;
7157 t->root = ZEND_JIT_TRACE_NUM;
7158 t->parent = 0;
7159 t->link = 0;
7160 t->exit_count = 0;
7161 t->child_count = 0;
7162 t->stack_map_size = 0;
7163 t->flags = 0;
7164 t->polymorphism = 0;
7165 t->jmp_table_size = 0;
7166 t->op_array = trace_buffer[0].op_array;
7167 t->opline = trace_buffer[1].opline;
7168 t->exit_info = exit_info;
7169 t->stack_map = NULL;
7170
7171 orig_trigger = JIT_G(trigger);
7172 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7173
7174 handler = zend_jit_trace(trace_buffer, 0, 0);
7175
7176 JIT_G(trigger) = orig_trigger;
7177
7178 if (handler) {
7179 zend_jit_trace_exit_info *shared_exit_info = NULL;
7180
7181 t->exit_info = NULL;
7182 if (t->exit_count) {
7183 /* reallocate exit_info into shared memory */
7184 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7185 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7186
7187 if (!shared_exit_info) {
7188 if (t->stack_map) {
7189 efree(t->stack_map);
7190 t->stack_map = NULL;
7191 }
7192 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7193 goto exit;
7194 }
7195 memcpy(shared_exit_info, exit_info,
7196 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7197 t->exit_info = shared_exit_info;
7198 }
7199
7200 if (t->stack_map_size) {
7201 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7202 if (!shared_stack_map) {
7203 efree(t->stack_map);
7204 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7205 goto exit;
7206 }
7207 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7208 efree(t->stack_map);
7209 t->stack_map = shared_stack_map;
7210 }
7211
7212 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7213 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7214
7215 ((zend_op*)opline)->handler = handler;
7216
7217 ZEND_JIT_TRACE_NUM++;
7218 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7219
7220 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7221 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7222 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7223 if (t->stack_map) {
7224 efree(t->stack_map);
7225 t->stack_map = NULL;
7226 }
7227 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7228 } else {
7229 if (t->stack_map) {
7230 efree(t->stack_map);
7231 t->stack_map = NULL;
7232 }
7233 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7234 }
7235
7236 exit:;
7237 } zend_catch {
7238 do_bailout = 1;
7239 } zend_end_try();
7240
7241 zend_jit_protect();
7242 SHM_PROTECT();
7243 }
7244
7245 zend_shared_alloc_unlock();
7246
7247 if (do_bailout) {
7248 zend_bailout();
7249 }
7250
7251 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7252 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7253 && t->exit_count > 0) {
7254 zend_jit_dump_exit_info(t);
7255 }
7256
7257 return ret;
7258 }
7259
7260 /* Set counting handler back to original VM handler. */
zend_jit_stop_hot_trace_counters(zend_op_array * op_array)7261 static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array)
7262 {
7263 zend_jit_op_array_trace_extension *jit_extension;
7264 uint32_t i;
7265
7266 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7267 for (i = 0; i < op_array->last; i++) {
7268 /* Opline with Jit-ed code handler is skipped. */
7269 if (jit_extension->trace_info[i].trace_flags &
7270 (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7271 continue;
7272 }
7273 if (jit_extension->trace_info[i].trace_flags &
7274 (ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_START_RETURN)) {
7275 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7276 }
7277 }
7278 }
7279
7280 /* Get the tracing op_array. */
zend_jit_stop_persistent_op_array(zend_op_array * op_array)7281 static void zend_jit_stop_persistent_op_array(zend_op_array *op_array)
7282 {
7283 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
7284 if (!func_info) {
7285 return;
7286 }
7287 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
7288 zend_jit_stop_hot_trace_counters(op_array);
7289 }
7290 }
7291
7292 /* Get all op_arrays with counter handler. */
zend_jit_stop_persistent_script(zend_persistent_script * script)7293 static void zend_jit_stop_persistent_script(zend_persistent_script *script)
7294 {
7295 zend_class_entry *ce;
7296 zend_op_array *op_array;
7297
7298 zend_jit_stop_persistent_op_array(&script->script.main_op_array);
7299
7300 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
7301 zend_jit_stop_persistent_op_array(op_array);
7302 } ZEND_HASH_FOREACH_END();
7303
7304 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
7305 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
7306 if (op_array->type == ZEND_USER_FUNCTION) {
7307 zend_jit_stop_persistent_op_array(op_array);
7308 }
7309 } ZEND_HASH_FOREACH_END();
7310 } ZEND_HASH_FOREACH_END();
7311 }
7312
7313 /* Get all scripts which are accelerated by JIT */
zend_jit_stop_counter_handlers(void)7314 static void zend_jit_stop_counter_handlers(void)
7315 {
7316 zend_shared_alloc_lock();
7317 /* mprotect has an extreme overhead, avoid calls to it for every function. */
7318 SHM_UNPROTECT();
7319 for (uint32_t i = 0; i < ZCSG(hash).max_num_entries; i++) {
7320 zend_accel_hash_entry *cache_entry;
7321 for (cache_entry = ZCSG(hash).hash_table[i]; cache_entry; cache_entry = cache_entry->next) {
7322 zend_persistent_script *script;
7323 if (cache_entry->indirect) continue;
7324 script = (zend_persistent_script *)cache_entry->data;
7325 zend_jit_stop_persistent_script(script);
7326 }
7327 }
7328 SHM_PROTECT();
7329 zend_shared_alloc_unlock();
7330 }
7331
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7332 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7333 {
7334 zend_shared_alloc_lock();
7335
7336 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7337 SHM_UNPROTECT();
7338 zend_jit_unprotect();
7339
7340 ((zend_op*)opline)->handler =
7341 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7342
7343 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7344
7345 zend_jit_protect();
7346 SHM_PROTECT();
7347 }
7348
7349 zend_shared_alloc_unlock();
7350 }
7351
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7352 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7353 {
7354 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7355 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7356 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7357 uint32_t cache_slot = JIT_G(bad_root_slot);
7358 uint32_t i;
7359
7360 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7361 if (cache_opline[i] == opline) {
7362 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7363 cache_opline[i] = NULL;
7364 return 1;
7365 } else {
7366 #if 0
7367 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7368 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7369 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7370 }
7371 #endif
7372 cache_count[i]++;
7373 cache_stop[i] = stop;
7374 return 0;
7375 }
7376 }
7377 }
7378 i = cache_slot;
7379 cache_opline[i] = opline;
7380 cache_count[i] = 1;
7381 cache_stop[i] = stop;
7382 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7383 JIT_G(bad_root_slot) = cache_slot;
7384 return 0;
7385 }
7386
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7387 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7388 {
7389 zend_jit_trace_rec *p = trace_buffer;
7390 const zend_op_array *op_array;
7391 const zend_op *opline;
7392 uint32_t level = 1 + trace_buffer[0].level;
7393 int idx, len, i, v, vars_count, call_level;
7394
7395 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7396 op_array = p->op_array;
7397 p += ZEND_JIT_TRACE_START_REC_SIZE;
7398 idx = 0;
7399 call_level = 0;
7400
7401 if (tssa && tssa->var_info) {
7402 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7403 vars_count = op_array->last_var;
7404 } else {
7405 vars_count = op_array->last_var + op_array->T;
7406 }
7407 for (i = 0; i < vars_count; i++) {
7408 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7409 fprintf(stderr, " %*c;", level, ' ');
7410 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7411 fprintf(stderr, "\n");
7412 }
7413 }
7414 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7415 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7416 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7417 zend_ssa_phi *p = tssa->blocks[1].phis;
7418
7419 fprintf(stderr, "LOOP:\n");
7420
7421 while (p) {
7422 fprintf(stderr, " ;");
7423 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7424 fprintf(stderr, " = Phi(");
7425 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7426 fprintf(stderr, ", ");
7427 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7428 fprintf(stderr, ")\n");
7429 p = p->next;
7430 }
7431 }
7432 }
7433
7434 while (1) {
7435 if (p->op == ZEND_JIT_TRACE_VM) {
7436 uint8_t op1_type, op2_type, op3_type;
7437
7438 opline = p->opline;
7439 fprintf(stderr, "%04d%*c",
7440 (int)(opline - op_array->opcodes),
7441 level, ' ');
7442 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7443
7444 op1_type = p->op1_type;
7445 op2_type = p->op2_type;
7446 op3_type = p->op3_type;
7447 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7448 fprintf(stderr, " ;");
7449 if (op1_type != IS_UNKNOWN) {
7450 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7451 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7452 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7453 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7454 p++;
7455 fprintf(stderr, " op1(%sobject of class %s)", ref,
7456 ZSTR_VAL(p->ce->name));
7457 } else {
7458 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7459 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7460 }
7461 }
7462 if (op2_type != IS_UNKNOWN) {
7463 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7464 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7465 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7466 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7467 p++;
7468 fprintf(stderr, " op2(%sobject of class %s)", ref,
7469 ZSTR_VAL(p->ce->name));
7470 } else {
7471 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7472 fprintf(stderr, " op2(%s%s)", ref, type);
7473 }
7474 }
7475 if (op3_type != IS_UNKNOWN) {
7476 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7477 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7478 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7479 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7480 fprintf(stderr, " op3(%s%s)", ref, type);
7481 }
7482 }
7483 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7484 uint8_t val_type;
7485 const char *type;
7486
7487 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7488 fprintf(stderr, " ;");
7489 }
7490 p++;
7491 val_type = p->op1_type;
7492
7493 if (val_type == IS_UNDEF) {
7494 type = "undef";
7495 } else if (val_type == IS_REFERENCE) {
7496 type = "ref";
7497 } else {
7498 type = zend_get_type_by_const(val_type);
7499 }
7500 fprintf(stderr, " val(%s)", type);
7501 }
7502 fprintf(stderr, "\n");
7503 idx++;
7504
7505 len = zend_jit_trace_op_len(opline);
7506 while (len > 1) {
7507 opline++;
7508 fprintf(stderr, "%04d%*c;",
7509 (int)(opline - op_array->opcodes),
7510 level, ' ');
7511 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7512 idx++;
7513 len--;
7514 fprintf(stderr, "\n");
7515 }
7516 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7517 op_array = p->op_array;
7518 fprintf(stderr, " %*c>enter %s%s%s\n",
7519 level, ' ',
7520 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7521 op_array->scope ? "::" : "",
7522 op_array->function_name ?
7523 ZSTR_VAL(op_array->function_name) :
7524 ZSTR_VAL(op_array->filename));
7525 level++;
7526 if (tssa && tssa->var_info) {
7527 call_level++;
7528 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7529 vars_count = op_array->last_var;
7530 for (i = 0; i < vars_count; i++, v++) {
7531 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7532 fprintf(stderr, " %*c;", level, ' ');
7533 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7534 fprintf(stderr, "\n");
7535 }
7536 }
7537 }
7538 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7539 op_array = p->op_array;
7540 level--;
7541 fprintf(stderr, " %*c<back %s%s%s\n",
7542 level, ' ',
7543 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7544 op_array->scope ? "::" : "",
7545 op_array->function_name ?
7546 ZSTR_VAL(op_array->function_name) :
7547 ZSTR_VAL(op_array->filename));
7548 if (tssa && tssa->var_info) {
7549 if (call_level == 0) {
7550 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7551 vars_count = op_array->last_var + op_array->T;
7552 for (i = 0; i < vars_count; i++, v++) {
7553 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7554 fprintf(stderr, " %*c;", level, ' ');
7555 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7556 fprintf(stderr, "\n");
7557 }
7558 }
7559 } else {
7560 call_level--;
7561 }
7562 }
7563 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7564 if (p->func != (zend_function*)&zend_pass_function) {
7565 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7566 level, ' ',
7567 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7568 (p->func && p->func->common.scope) ? "::" : "",
7569 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7570 } else {
7571 fprintf(stderr, " %*c>skip\n",
7572 level, ' ');
7573 }
7574 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7575 if (p->func != (zend_function*)&zend_pass_function) {
7576 fprintf(stderr, " %*c>call %s%s%s\n",
7577 level, ' ',
7578 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7579 p->func->common.scope ? "::" : "",
7580 ZSTR_VAL(p->func->common.function_name));
7581 } else {
7582 fprintf(stderr, " %*c>skip\n",
7583 level, ' ');
7584 }
7585 } else if (p->op == ZEND_JIT_TRACE_END) {
7586 break;
7587 }
7588 p++;
7589 }
7590 }
7591
zend_jit_dump_exit_info(zend_jit_trace_info * t)7592 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7593 {
7594 int i, j;
7595
7596 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7597 for (i = 0; i < t->exit_count; i++) {
7598 const zend_op_array *op_array = t->exit_info[i].op_array;
7599 uint32_t stack_size = t->exit_info[i].stack_size;
7600 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset;
7601
7602 fprintf(stderr, " exit_%d:", i);
7603 if (t->exit_info[i].opline) {
7604 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7605 } else {
7606 fprintf(stderr, " ----/");
7607 }
7608 if (t->exit_info[i].stack_size) {
7609 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7610 } else {
7611 fprintf(stderr, "----/0");
7612 }
7613 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7614 fprintf(stderr, "/VM");
7615 }
7616 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7617 fprintf(stderr, "/CALL");
7618 }
7619 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7620 fprintf(stderr, "/POLY");
7621 }
7622 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7623 fprintf(stderr, "/FREE_OP1");
7624 }
7625 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7626 fprintf(stderr, "/FREE_OP2");
7627 }
7628 for (j = 0; j < stack_size; j++) {
7629 zend_uchar type = STACK_TYPE(stack, j);
7630 if (type != IS_UNKNOWN) {
7631 fprintf(stderr, " ");
7632 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7633 fprintf(stderr, ":");
7634 if (type == IS_UNDEF) {
7635 fprintf(stderr, "undef");
7636 } else {
7637 fprintf(stderr, "%s", zend_get_type_by_const(type));
7638 }
7639 if (STACK_REG(stack, j) != ZREG_NONE) {
7640 if (STACK_REG(stack, j) < ZREG_NUM) {
7641 fprintf(stderr, "(%s)", zend_reg_name[STACK_REG(stack, j)]);
7642 } else if (STACK_REG(stack, j) == ZREG_THIS) {
7643 fprintf(stderr, "(this)");
7644 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7645 fprintf(stderr, "(zval_try_addref)");
7646 } else {
7647 fprintf(stderr, "(const_%d)", STACK_REG(stack, j) - ZREG_NUM);
7648 }
7649 }
7650 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7651 fprintf(stderr, " ");
7652 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7653 fprintf(stderr, ":unknown(zval_try_addref)");
7654 } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_GPR0) {
7655 fprintf(stderr, " ");
7656 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7657 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[ZREG_COPY]);
7658 }
7659 }
7660 fprintf(stderr, "\n");
7661 }
7662 }
7663
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)7664 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
7665 {
7666 const zend_op *orig_opline;
7667 zend_jit_trace_stop stop;
7668 int ret = 0;
7669 zend_op_array *op_array;
7670 zend_jit_op_array_trace_extension *jit_extension;
7671 size_t offset;
7672 uint32_t trace_num;
7673 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7674
7675 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
7676 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
7677 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
7678
7679 repeat:
7680 trace_num = ZEND_JIT_TRACE_NUM;
7681 orig_opline = opline;
7682 op_array = &EX(func)->op_array;
7683 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7684 offset = jit_extension->offset;
7685
7686 EX(opline) = opline;
7687
7688 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
7689 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7690 return 0;
7691 }
7692
7693 if (JIT_G(tracing)) {
7694 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
7695 return 0;
7696 }
7697
7698 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7699 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7700 trace_num,
7701 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7702 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7703 EX(func)->op_array.scope ? "::" : "",
7704 EX(func)->op_array.function_name ?
7705 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7706 ZSTR_VAL(EX(func)->op_array.filename),
7707 opline->lineno);
7708 }
7709
7710 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7711 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7712 zend_jit_stop_counter_handlers();
7713 goto abort;
7714 }
7715
7716 JIT_G(tracing) = 1;
7717 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
7718 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
7719 JIT_G(tracing) = 0;
7720
7721 if (stop & ZEND_JIT_TRACE_HALT) {
7722 ret = -1;
7723 }
7724 stop &= ~ZEND_JIT_TRACE_HALT;
7725
7726 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
7727 orig_opline = trace_buffer[1].opline;
7728 op_array = (zend_op_array*)trace_buffer[0].op_array;
7729 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7730 offset = jit_extension->offset;
7731 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7732 const zend_op_array *op_array = trace_buffer[0].op_array;
7733 const zend_op *opline = trace_buffer[1].opline;
7734 zend_jit_op_array_trace_extension *jit_extension =
7735 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7736 size_t offset = jit_extension->offset;
7737
7738 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7739 trace_num,
7740 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7741 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7742 op_array->scope ? "::" : "",
7743 op_array->function_name ?
7744 ZSTR_VAL(op_array->function_name) : "$main",
7745 ZSTR_VAL(op_array->filename),
7746 opline->lineno);
7747 }
7748 }
7749
7750 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
7751 zend_jit_dump_trace(trace_buffer, NULL);
7752 }
7753
7754 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
7755 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
7756 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
7757 uint32_t idx = trace_buffer[1].last;
7758 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
7759 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
7760 trace_num,
7761 link_to);
7762 } else {
7763 fprintf(stderr, "---- TRACE %d stop (%s)\n",
7764 trace_num,
7765 zend_jit_trace_stop_description[stop]);
7766 }
7767 }
7768 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
7769 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
7770 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
7771 fprintf(stderr, "---- TRACE %d %s\n",
7772 trace_num,
7773 zend_jit_trace_stop_description[stop]);
7774 }
7775 } else {
7776 goto abort;
7777 }
7778 } else {
7779 abort:
7780 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
7781 fprintf(stderr, "---- TRACE %d abort (%s)\n",
7782 trace_num,
7783 zend_jit_trace_stop_description[stop]);
7784 }
7785 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
7786 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
7787 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7788 fprintf(stderr, "---- TRACE %d blacklisted\n",
7789 trace_num);
7790 }
7791 zend_jit_blacklist_root_trace(orig_opline, offset);
7792 }
7793 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
7794 execute_data = EG(current_execute_data);
7795 opline = EX(opline);
7796 goto repeat;
7797 }
7798 }
7799
7800 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
7801 fprintf(stderr, "\n");
7802 }
7803
7804 return ret;
7805 }
7806
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)7807 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
7808 {
7809 const void *handler;
7810 bool do_bailout = 0;
7811
7812 zend_shared_alloc_lock();
7813
7814 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
7815 SHM_UNPROTECT();
7816 zend_jit_unprotect();
7817
7818 zend_try {
7819 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
7820
7821 if (handler) {
7822 zend_jit_link_side_trace(
7823 zend_jit_traces[trace_num].code_start,
7824 zend_jit_traces[trace_num].code_size,
7825 zend_jit_traces[trace_num].jmp_table_size,
7826 exit_num,
7827 handler);
7828 }
7829 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
7830 } zend_catch {
7831 do_bailout = 1;
7832 } zend_end_try();
7833
7834 zend_jit_protect();
7835 SHM_PROTECT();
7836 }
7837
7838 zend_shared_alloc_unlock();
7839
7840 if (do_bailout) {
7841 zend_bailout();
7842 }
7843 }
7844
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)7845 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
7846 {
7847 uint8_t *counter = JIT_G(exit_counters) +
7848 zend_jit_traces[trace_num].exit_counters + exit_num;
7849
7850 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
7851 return 1;
7852 }
7853 (*counter)++;
7854 return 0;
7855 }
7856
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)7857 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
7858 {
7859 uint8_t *counter = JIT_G(exit_counters) +
7860 zend_jit_traces[trace_num].exit_counters + exit_num;
7861
7862 if (*counter + 1 >= JIT_G(hot_side_exit)) {
7863 return 1;
7864 }
7865 (*counter)++;
7866 return 0;
7867 }
7868
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)7869 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
7870 {
7871 zend_jit_trace_stop ret;
7872 const void *handler;
7873 uint8_t orig_trigger;
7874 zend_jit_trace_info *t;
7875 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7876 bool do_bailout = 0;
7877
7878 zend_shared_alloc_lock();
7879
7880 /* Checks under lock */
7881 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7882 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7883 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7884 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7885 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
7886 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
7887 } else {
7888 SHM_UNPROTECT();
7889 zend_jit_unprotect();
7890
7891 zend_try {
7892 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7893
7894 t->id = ZEND_JIT_TRACE_NUM;
7895 t->root = zend_jit_traces[parent_num].root;
7896 t->parent = parent_num;
7897 t->link = 0;
7898 t->exit_count = 0;
7899 t->child_count = 0;
7900 t->stack_map_size = 0;
7901 t->flags = 0;
7902 t->polymorphism = polymorphism;
7903 t->jmp_table_size = 0;
7904 t->opline = NULL;
7905 t->exit_info = exit_info;
7906 t->stack_map = NULL;
7907
7908 orig_trigger = JIT_G(trigger);
7909 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7910
7911 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
7912
7913 JIT_G(trigger) = orig_trigger;
7914
7915 if (handler) {
7916 zend_jit_trace_exit_info *shared_exit_info = NULL;
7917
7918 t->exit_info = NULL;
7919 if (t->exit_count) {
7920 /* reallocate exit_info into shared memory */
7921 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7922 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7923
7924 if (!shared_exit_info) {
7925 if (t->stack_map) {
7926 efree(t->stack_map);
7927 t->stack_map = NULL;
7928 }
7929 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7930 goto exit;
7931 }
7932 memcpy(shared_exit_info, exit_info,
7933 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7934 t->exit_info = shared_exit_info;
7935 }
7936
7937 if (t->stack_map_size) {
7938 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7939 if (!shared_stack_map) {
7940 efree(t->stack_map);
7941 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7942 goto exit;
7943 }
7944 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7945 efree(t->stack_map);
7946 t->stack_map = shared_stack_map;
7947 }
7948
7949 zend_jit_link_side_trace(
7950 zend_jit_traces[parent_num].code_start,
7951 zend_jit_traces[parent_num].code_size,
7952 zend_jit_traces[parent_num].jmp_table_size,
7953 exit_num,
7954 handler);
7955
7956 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7957 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7958
7959 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
7960 ZEND_JIT_TRACE_NUM++;
7961 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
7962
7963 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7964 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7965 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7966 if (t->stack_map) {
7967 efree(t->stack_map);
7968 t->stack_map = NULL;
7969 }
7970 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7971 } else {
7972 if (t->stack_map) {
7973 efree(t->stack_map);
7974 t->stack_map = NULL;
7975 }
7976 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7977 }
7978
7979 exit:;
7980 } zend_catch {
7981 do_bailout = 1;
7982 } zend_end_try();
7983
7984 zend_jit_protect();
7985 SHM_PROTECT();
7986 }
7987
7988 zend_shared_alloc_unlock();
7989
7990 if (do_bailout) {
7991 zend_bailout();
7992 }
7993
7994 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7995 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7996 && t->exit_count > 0) {
7997 zend_jit_dump_exit_info(t);
7998 }
7999
8000 return ret;
8001 }
8002
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)8003 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
8004 {
8005 zend_jit_trace_stop stop;
8006 int ret = 0;
8007 uint32_t trace_num;
8008 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8009 uint32_t is_megamorphic = 0;
8010 uint32_t polymorphism = 0;
8011
8012 trace_num = ZEND_JIT_TRACE_NUM;
8013
8014 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8015 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8016 return 0;
8017 }
8018
8019 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8020 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
8021 trace_num, parent_num, exit_num,
8022 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8023 EX(func)->op_array.scope ? "::" : "",
8024 EX(func)->op_array.function_name ?
8025 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8026 ZSTR_VAL(EX(func)->op_array.filename),
8027 EX(opline)->lineno);
8028 }
8029
8030 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8031 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8032 goto abort;
8033 }
8034
8035 if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8036 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8037 goto abort;
8038 }
8039
8040 if (JIT_G(max_polymorphic_calls) > 0) {
8041 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
8042 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
8043 && EX(call))) {
8044 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
8045 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
8046 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
8047 } else if (!zend_jit_traces[parent_num].polymorphism) {
8048 polymorphism = 1;
8049 } else if (exit_num == 0) {
8050 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
8051 }
8052 }
8053 }
8054
8055 JIT_G(tracing) = 1;
8056 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
8057 JIT_G(tracing) = 0;
8058
8059 if (stop & ZEND_JIT_TRACE_HALT) {
8060 ret = -1;
8061 }
8062 stop &= ~ZEND_JIT_TRACE_HALT;
8063
8064 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8065 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8066 const zend_op_array *op_array = trace_buffer[0].op_array;
8067 const zend_op *opline = trace_buffer[1].opline;
8068 zend_jit_op_array_trace_extension *jit_extension =
8069 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8070 size_t offset = jit_extension->offset;
8071
8072 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8073 trace_num,
8074 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8075 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8076 op_array->scope ? "::" : "",
8077 op_array->function_name ?
8078 ZSTR_VAL(op_array->function_name) : "$main",
8079 ZSTR_VAL(op_array->filename),
8080 opline->lineno);
8081 }
8082 }
8083
8084 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8085 zend_jit_dump_trace(trace_buffer, NULL);
8086 }
8087
8088 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8089 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8090 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8091 uint32_t idx = trace_buffer[1].last;
8092 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8093 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8094 trace_num,
8095 link_to);
8096 } else {
8097 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8098 trace_num,
8099 zend_jit_trace_stop_description[stop]);
8100 }
8101 }
8102 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8103 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8104 } else {
8105 const zend_op_array *op_array = trace_buffer[0].op_array;
8106 zend_jit_op_array_trace_extension *jit_extension =
8107 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8108 const zend_op *opline = trace_buffer[1].opline;
8109
8110 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8111 }
8112 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8113 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8114 fprintf(stderr, "---- TRACE %d %s\n",
8115 trace_num,
8116 zend_jit_trace_stop_description[stop]);
8117 }
8118 } else {
8119 goto abort;
8120 }
8121 } else {
8122 abort:
8123 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8124 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8125 trace_num,
8126 zend_jit_trace_stop_description[stop]);
8127 }
8128 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8129 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8130 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8131 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8132 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8133 parent_num, exit_num);
8134 }
8135 }
8136 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8137 execute_data = EG(current_execute_data);
8138 return zend_jit_trace_hot_root(execute_data, EX(opline));
8139 }
8140 }
8141
8142 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8143 fprintf(stderr, "\n");
8144 }
8145
8146 return ret;
8147 }
8148
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8149 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8150 {
8151 uint32_t trace_num = EG(jit_trace_num);
8152 zend_execute_data *execute_data = EG(current_execute_data);
8153 const zend_op *orig_opline = EX(opline);
8154 const zend_op *opline;
8155 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8156 int repeat_last_opline = 0;
8157
8158 /* Deoptimization of VM stack state */
8159 uint32_t i;
8160 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8161 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset;
8162
8163 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8164 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8165 call->prev_execute_data = EX(call);
8166 EX(call) = call;
8167 }
8168
8169 for (i = 0; i < stack_size; i++) {
8170 if (STACK_REG(stack, i) != ZREG_NONE) {
8171 if (STACK_TYPE(stack, i) == IS_LONG) {
8172 zend_long val;
8173
8174 if (STACK_REG(stack, i) < ZREG_NUM) {
8175 val = regs->gpr[STACK_REG(stack, i)];
8176 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN) {
8177 val = ZEND_LONG_MIN;
8178 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX) {
8179 val = ZEND_LONG_MAX;
8180 } else {
8181 ZEND_UNREACHABLE();
8182 }
8183 ZVAL_LONG(EX_VAR_NUM(i), val);
8184 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8185 double val;
8186
8187 if (STACK_REG(stack, i) < ZREG_NUM) {
8188 val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8189 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN_MINUS_1) {
8190 val = (double)ZEND_LONG_MIN - 1.0;
8191 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX_PLUS_1) {
8192 val = (double)ZEND_LONG_MAX + 1.0;
8193 } else {
8194 ZEND_UNREACHABLE();
8195 }
8196 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8197 } else if (STACK_REG(stack, i) == ZREG_THIS) {
8198 zend_object *obj = Z_OBJ(EX(This));
8199
8200 GC_ADDREF(obj);
8201 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8202 } else if (STACK_REG(stack, i) == ZREG_NULL) {
8203 ZVAL_NULL(EX_VAR_NUM(i));
8204 } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) {
8205 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8206 } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_GPR0) {
8207 zval *val = (zval*)regs->gpr[ZREG_COPY];
8208
8209 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8210 /* Undefined array index or property */
8211 repeat_last_opline = 1;
8212 } else {
8213 ZVAL_COPY(EX_VAR_NUM(i), val);
8214 }
8215 } else {
8216 ZEND_UNREACHABLE();
8217 }
8218 }
8219 }
8220
8221 if (repeat_last_opline) {
8222 EX(opline) = t->exit_info[exit_num].opline - 1;
8223 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8224 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8225 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8226 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8227 }
8228 return 1;
8229 }
8230
8231 opline = t->exit_info[exit_num].opline;
8232
8233 if (opline) {
8234 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8235 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8236 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8237 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8238 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8239 EX(opline) = opline-1;
8240 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8241 }
8242 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8243 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8244 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8245 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8246 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8247 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8248 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8249 EX(opline) = opline-1;
8250 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8251 }
8252 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8253 if (EG(exception)) {
8254 return 1;
8255 }
8256 }
8257 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8258 zend_function *func = (zend_function*)regs->gpr[ZREG_COPY];
8259
8260 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8261 zend_string_release_ex(func->common.function_name, 0);
8262 zend_free_trampoline(func);
8263 EX(opline) = opline;
8264 return 1;
8265 }
8266 }
8267
8268 /* Set VM opline to continue interpretation */
8269 EX(opline) = opline;
8270 }
8271
8272 if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) {
8273 return 1;
8274 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8275 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8276 return 0;
8277 }
8278
8279 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8280 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8281 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8282
8283 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8284 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8285 trace_num,
8286 exit_num,
8287 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8288 EX(func)->op_array.scope ? "::" : "",
8289 EX(func)->op_array.function_name ?
8290 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8291 ZSTR_VAL(EX(func)->op_array.filename),
8292 EX(opline)->lineno);
8293 }
8294
8295 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8296 zend_jit_op_array_trace_extension *jit_extension;
8297 uint32_t num = trace_num;
8298
8299 while (t->root != num) {
8300 num = t->root;
8301 t = &zend_jit_traces[num];
8302 }
8303
8304 zend_shared_alloc_lock();
8305
8306 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8307
8308 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8309 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8310 /* skip: not JIT-ed nor blacklisted */
8311 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8312 /* too many root traces, blacklist the root trace */
8313 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
8314 SHM_UNPROTECT();
8315 zend_jit_unprotect();
8316
8317 ((zend_op*)opline)->handler =
8318 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->orig_handler;
8319
8320 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= ~ZEND_JIT_TRACE_JITED;
8321 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
8322
8323 zend_jit_protect();
8324 SHM_PROTECT();
8325 }
8326 } else {
8327 SHM_UNPROTECT();
8328 zend_jit_unprotect();
8329
8330 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8331 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8332 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8333 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8334 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8335 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8336 }
8337 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8338 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8339
8340 zend_jit_protect();
8341 SHM_PROTECT();
8342 }
8343
8344 zend_shared_alloc_unlock();
8345
8346 return 0;
8347 }
8348
8349 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8350 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8351 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8352 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8353 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8354 trace_num, exit_num);
8355 }
8356 return 0;
8357 }
8358 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8359 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8360 }
8361
8362 /* Return 1 to call original handler instead of the same JIT-ed trace */
8363 return (orig_opline == t->opline && EX(opline) == orig_opline);
8364 }
8365
zend_jit_trace_supported(const zend_op * opline)8366 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8367 {
8368 switch (opline->opcode) {
8369 case ZEND_CATCH:
8370 case ZEND_FAST_CALL:
8371 case ZEND_FAST_RET:
8372 return ZEND_JIT_TRACE_UNSUPPORTED;
8373 default:
8374 return ZEND_JIT_TRACE_SUPPORTED;
8375 }
8376 }
8377
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8378 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8379 {
8380 zend_jit_op_array_trace_extension *jit_extension;
8381 uint32_t i;
8382
8383 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8384 for (i = 0; i < op_array->last; i++) {
8385 jit_extension->trace_info[i].trace_flags &=
8386 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8387 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8388 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8389 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8390 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8391 } else {
8392 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8393 }
8394 }
8395 return SUCCESS;
8396 }
8397
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8398 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8399 {
8400 zend_op *opline;
8401 zend_jit_op_array_trace_extension *jit_extension;
8402 uint32_t i;
8403
8404 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8405
8406 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8407 if (!jit_extension) {
8408 return FAILURE;
8409 }
8410 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8411 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8412 jit_extension->op_array = op_array;
8413 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8414 for (i = 0; i < op_array->last; i++) {
8415 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8416 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8417 jit_extension->trace_info[i].counter = NULL;
8418 jit_extension->trace_info[i].trace_flags =
8419 zend_jit_trace_supported(&op_array->opcodes[i]);
8420 }
8421 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8422
8423 if (JIT_G(hot_loop)) {
8424 zend_cfg cfg;
8425
8426 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8427
8428 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8429 return FAILURE;
8430 }
8431
8432 for (i = 0; i < cfg.blocks_count; i++) {
8433 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8434 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8435 /* loop header */
8436 opline = op_array->opcodes + cfg.blocks[i].start;
8437 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8438 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8439 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8440 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8441 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8442 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8443 }
8444 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8445 ZEND_JIT_TRACE_START_LOOP;
8446 }
8447 }
8448 }
8449 }
8450 }
8451
8452 if (JIT_G(hot_func)) {
8453 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8454 opline = op_array->opcodes;
8455 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8456 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8457 opline++;
8458 }
8459 }
8460
8461 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8462 /* function entry */
8463 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8464 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8465 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8466 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8467 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8468 ZEND_JIT_TRACE_START_ENTER;
8469 }
8470 }
8471
8472 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8473
8474 return SUCCESS;
8475 }
8476
zend_jit_trace_init_caches(void)8477 static void zend_jit_trace_init_caches(void)
8478 {
8479 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8480 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8481 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8482 JIT_G(bad_root_slot) = 0;
8483
8484 if (JIT_G(exit_counters)) {
8485 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8486 }
8487 }
8488
zend_jit_trace_reset_caches(void)8489 static void zend_jit_trace_reset_caches(void)
8490 {
8491 JIT_G(tracing) = 0;
8492 #ifdef ZTS
8493 if (!JIT_G(exit_counters)) {
8494 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8495 }
8496 #endif
8497 }
8498
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8499 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8500 {
8501 if (jit_globals->exit_counters) {
8502 free(jit_globals->exit_counters);
8503 }
8504 }
8505
zend_jit_trace_restart(void)8506 static void zend_jit_trace_restart(void)
8507 {
8508 ZEND_JIT_TRACE_NUM = 1;
8509 ZEND_JIT_COUNTER_NUM = 0;
8510 ZEND_JIT_EXIT_NUM = 0;
8511 ZEND_JIT_EXIT_COUNTERS = 0;
8512
8513 zend_jit_trace_init_caches();
8514 }
8515
