1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(bool reattached)49 static int zend_jit_trace_startup(bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 return FAILURE;
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 return FAILURE;
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 ZCSG(jit_counters_stopped) = false;
67 } else {
68 zend_jit_traces = ZCSG(jit_traces);
69 if (!zend_jit_traces) {
70 return FAILURE;
71 }
72 zend_jit_exit_groups = ZCSG(jit_exit_groups);
73 if (!zend_jit_exit_groups) {
74 return FAILURE;
75 }
76 }
77
78 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
79 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
80
81 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
82 if (JIT_G(exit_counters) == NULL) {
83 return FAILURE;
84 }
85
86 return SUCCESS;
87 }
88
zend_jit_trace_allocate_exit_group(uint32_t n)89 static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
90 {
91 dasm_State* dasm_state = NULL;
92 const void *entry;
93 char name[32];
94
95 dasm_init(&dasm_state, DASM_MAXSECTION);
96 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
97 dasm_setup(&dasm_state, dasm_actions);
98 zend_jit_align_stub(&dasm_state);
99 zend_jit_trace_exit_group_stub(&dasm_state, n);
100
101 sprintf(name, "jit$$trace_exit_%d", n);
102 entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE);
103 dasm_free(&dasm_state);
104
105 #ifdef HAVE_DISASM
106 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
107 uint32_t i;
108
109 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
110 sprintf(name, "jit$$trace_exit_%d", n + i);
111 zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
112 }
113 }
114 #endif
115
116 return entry;
117 }
118
zend_jit_trace_allocate_exit_point(uint32_t n)119 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
120 {
121 const void *group = NULL;
122
123 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
124 return NULL;
125 }
126 do {
127 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
128 if (!group) {
129 return NULL;
130 }
131 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
132 group;
133 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
134 } while (n >= ZEND_JIT_EXIT_NUM);
135 return (const void*)
136 ((const char*)group +
137 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
138 }
139
zend_jit_trace_get_exit_addr(uint32_t n)140 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
141 {
142 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
143 return zend_jit_trace_allocate_exit_point(n);
144 }
145 return (const void*)
146 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
147 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
148 }
149
150 #if ZEND_JIT_TARGET_ARM64
zend_jit_get_current_trace_info(void)151 static zend_jit_trace_info *zend_jit_get_current_trace_info(void)
152 {
153 return &zend_jit_traces[ZEND_JIT_TRACE_NUM];
154 }
155
zend_jit_trace_find_exit_point(const void * addr)156 static uint32_t zend_jit_trace_find_exit_point(const void* addr)
157 {
158 uint32_t n = ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP;
159 uint32_t i;
160
161 for (i = 0; i < n; i++) {
162 if ((const char*)addr >= (const char*)zend_jit_exit_groups[i]
163 && (const char*)addr < (const char*)zend_jit_exit_groups[i] +
164 (ZEND_JIT_EXIT_POINTS_PER_GROUP * ZEND_JIT_EXIT_POINTS_SPACING)) {
165 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
166 ((const char*)addr - (const char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING;
167 }
168 }
169 return (uint32_t)-1;
170 }
171 #endif
172
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)173 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
174 {
175 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
176 uint32_t exit_point;
177 const zend_op_array *op_array;
178 uint32_t stack_offset = (uint32_t)-1;
179 uint32_t stack_size;
180 zend_jit_trace_stack *stack = NULL;
181
182 if (delayed_call_chain) {
183 assert(to_opline != NULL); /* CALL and IP share the same register */
184 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
185 }
186 if (JIT_G(current_frame)) {
187 op_array = &JIT_G(current_frame)->func->op_array;
188 stack_size = op_array->last_var + op_array->T;
189 if (stack_size) {
190 stack = JIT_G(current_frame)->stack;
191 do {
192 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
193 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
194 || STACK_REG(stack, stack_size-1) != ZREG_NONE) {
195 break;
196 }
197 stack_size--;
198 } while (stack_size);
199 }
200 } else {
201 op_array = NULL;
202 stack_size = 0;
203 }
204
205 /* Try to reuse exit points */
206 if (to_opline != NULL && t->exit_count > 0) {
207 uint32_t i = t->exit_count;
208
209 do {
210 i--;
211 if (stack_size == 0
212 || (t->exit_info[i].stack_size >= stack_size
213 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
214 stack_offset = t->exit_info[i].stack_offset;
215 if (t->exit_info[i].opline == to_opline
216 && t->exit_info[i].flags == flags
217 && t->exit_info[i].stack_size == stack_size) {
218 return i;
219 }
220 }
221 } while (i > 0);
222 }
223
224 exit_point = t->exit_count;
225 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
226 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
227 stack_offset = t->stack_map_size;
228 t->stack_map_size += stack_size;
229 // TODO: reduce number of reallocations ???
230 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
231 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
232 }
233 t->exit_count++;
234 t->exit_info[exit_point].opline = to_opline;
235 t->exit_info[exit_point].op_array = op_array;
236 t->exit_info[exit_point].flags = flags;
237 t->exit_info[exit_point].stack_size = stack_size;
238 t->exit_info[exit_point].stack_offset = stack_offset;
239 }
240
241 return exit_point;
242 }
243
zend_jit_trace_add_code(const void * start,uint32_t size)244 static void zend_jit_trace_add_code(const void *start, uint32_t size)
245 {
246 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
247
248 t->code_start = start;
249 t->code_size = size;
250 }
251
252 /**
253 * Locate a trace in the #zend_jit_traces array with the specified
254 * #code_start address.
255 *
256 * @return the #zend_jit_traces index or 0 if no such #code_start
257 * address was found
258 */
zend_jit_find_trace(const void * addr)259 static uint32_t zend_jit_find_trace(const void *addr)
260 {
261 uint32_t i;
262
263 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
264 if (zend_jit_traces[i].code_start == addr) {
265 return i;
266 }
267 }
268 return 0;
269 }
270
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)271 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
272 {
273 smart_str buf = {0};
274
275 smart_str_appends(&buf, TRACE_PREFIX);
276 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
277 smart_str_appendc(&buf, '$');
278 if (op_array->function_name) {
279 if (op_array->scope) {
280 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
281 smart_str_appends(&buf, "::");
282 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
283 } else {
284 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
285 }
286 } else if (op_array->filename) {
287 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
288 }
289 smart_str_appendc(&buf, '$');
290 smart_str_append_long(&buf, (zend_long)lineno);
291 smart_str_0(&buf);
292 return buf.s;
293 }
294
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)295 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
296 {
297 switch (opline->opcode) {
298 case ZEND_IS_IDENTICAL:
299 case ZEND_IS_NOT_IDENTICAL:
300 case ZEND_IS_EQUAL:
301 case ZEND_IS_NOT_EQUAL:
302 case ZEND_IS_SMALLER:
303 case ZEND_IS_SMALLER_OR_EQUAL:
304 case ZEND_CASE:
305 case ZEND_CASE_STRICT:
306 case ZEND_ISSET_ISEMPTY_CV:
307 case ZEND_ISSET_ISEMPTY_VAR:
308 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
309 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
310 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
311 case ZEND_INSTANCEOF:
312 case ZEND_TYPE_CHECK:
313 case ZEND_DEFINED:
314 case ZEND_IN_ARRAY:
315 case ZEND_ARRAY_KEY_EXISTS:
316 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
317 /* smart branch */
318 return 1;
319 }
320 break;
321 case ZEND_JMPZ:
322 case ZEND_JMPNZ:
323 case ZEND_JMPZ_EX:
324 case ZEND_JMPNZ_EX:
325 case ZEND_JMP_SET:
326 case ZEND_COALESCE:
327 case ZEND_JMP_NULL:
328 case ZEND_FE_RESET_R:
329 case ZEND_FE_RESET_RW:
330 case ZEND_ASSERT_CHECK:
331 case ZEND_FE_FETCH_R:
332 case ZEND_FE_FETCH_RW:
333 case ZEND_SWITCH_LONG:
334 case ZEND_SWITCH_STRING:
335 case ZEND_MATCH:
336 case ZEND_BIND_INIT_STATIC_OR_JMP:
337 /* branch opcodes */
338 return 1;
339 case ZEND_NEW:
340 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
341 /* NEW may skip constructor without arguments */
342 return 1;
343 }
344 break;
345 case ZEND_CATCH:
346 case ZEND_FAST_CALL:
347 case ZEND_FAST_RET:
348 case ZEND_GENERATOR_CREATE:
349 case ZEND_GENERATOR_RETURN:
350 case ZEND_EXIT:
351 case ZEND_YIELD:
352 case ZEND_YIELD_FROM:
353 case ZEND_INCLUDE_OR_EVAL:
354 case ZEND_MATCH_ERROR:
355 /* unsupported */
356 return 1;
357 case ZEND_DO_FCALL:
358 /* potentially polymorphic call */
359 return 1;
360 #if 0
361 case ZEND_DO_UCALL:
362 case ZEND_DO_FCALL_BY_NAME:
363 /* monomorphic call */
364 // TODO: recompilation may change target ???
365 return 0;
366 #endif
367 case ZEND_RETURN_BY_REF:
368 case ZEND_RETURN:
369 /* return */
370 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
371 default:
372 break;
373 }
374 return 0;
375 }
376
zend_jit_trace_type_to_info_ex(uint8_t type,uint32_t info)377 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(uint8_t type, uint32_t info)
378 {
379 if (type == IS_UNKNOWN) {
380 return info;
381 }
382 ZEND_ASSERT(info & (1 << type));
383 if (type < IS_STRING) {
384 return (1 << type);
385 } else if (type != IS_ARRAY) {
386 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
387 } else {
388 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
389 }
390 }
391
zend_jit_trace_type_to_info(uint8_t type)392 static zend_always_inline uint32_t zend_jit_trace_type_to_info(uint8_t type)
393 {
394 return zend_jit_trace_type_to_info_ex(type, -1);
395 }
396
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)397 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
398 {
399 if (var >= op_array->last_var) {
400 return NO_ALIAS;
401 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
402 return SYMTABLE_ALIAS;
403 } else if (ssa->vars) {
404 return ssa->vars[var].alias;
405 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
406 return HTTP_RESPONSE_HEADER_ALIAS;
407 }
408 return NO_ALIAS;
409 }
410
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)411 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
412 int ssa_var,
413 uint8_t op_type)
414 {
415 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
416
417 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
418 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
419 info->type |= MAY_BE_GUARD;
420 } else {
421 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
422 }
423 }
424 }
425
426 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
427 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
428 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
429 } \
430 } while (0)
431
432 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
433 if (op_type != IS_UNKNOWN) { \
434 if ((op_info & MAY_BE_GUARD) != 0) { \
435 if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \
436 goto jit_failure; \
437 } \
438 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
439 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
440 op_info = zend_jit_trace_type_to_info(op_type); \
441 } else { \
442 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
443 op_info &= ~MAY_BE_GUARD; \
444 ssa->var_info[_ssa_var].type &= op_info; \
445 } \
446 } \
447 } \
448 } while (0)
449
450 #define ADD_OP1_TRACE_GUARD() \
451 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
452 #define ADD_OP2_TRACE_GUARD() \
453 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
454 #define ADD_OP1_DATA_TRACE_GUARD() \
455 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
456
457 #define CHECK_OP1_TRACE_TYPE() \
458 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
459 #define CHECK_OP2_TRACE_TYPE() \
460 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
461 #define CHECK_OP1_DATA_TRACE_TYPE() \
462 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
463
zend_jit_trace_frame_size(const zend_op_array * op_array)464 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
465 {
466 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
467 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
468 } else if (op_array) {
469 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
470 } else {
471 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
472 }
473 }
474
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)475 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
476 {
477 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
478 }
479
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)480 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
481 {
482 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
483 }
484
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,uint8_t type)485 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type)
486 {
487 zend_jit_trace_stack *stack = call->stack;
488 const zend_op_array *op_array = &call->func->op_array;
489 uint32_t arg_num = opline->op2.num;
490
491 if (arg_num > op_array->num_args) {
492 return;
493 }
494 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
495 zend_arg_info *arg_info;
496
497 ZEND_ASSERT(arg_num <= op_array->num_args);
498 arg_info = &op_array->arg_info[arg_num-1];
499
500 if (ZEND_TYPE_IS_SET(arg_info->type)) {
501 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
502 return;
503 }
504 }
505 }
506 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
507 }
508
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)509 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
510 {
511 if (func
512 && func->type == ZEND_INTERNAL_FUNCTION
513 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
514 && arg_num < func->internal_function.num_args) {
515 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
516
517 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
518 && ZEND_TYPE_IS_SET(arg_info->type)
519 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
520 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
521 && JIT_G(current_frame)
522 && JIT_G(current_frame)->call
523 && JIT_G(current_frame)->call->func) {
524 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
525
526 if (type != IS_UNKNOWN
527 && type < IS_STRING
528 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
529 return 0;
530 }
531 }
532 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
533 const zend_op *opline = call_info->arg_info[arg_num].opline;
534
535 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
536 zval *zv = RT_CONSTANT(opline, opline->op1);
537
538 if (!Z_REFCOUNTED_P(zv)) {
539 uint32_t type = Z_TYPE_P(zv);
540
541 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
542 if (type != IS_ARRAY
543 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
544 return 0;
545 }
546 }
547 }
548 }
549 }
550 }
551
552 return 1;
553 }
554
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)555 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
556 {
557 zend_jit_op_array_trace_extension *jit_extension;
558 zend_ssa *ssa;
559
560 jit_extension =
561 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
562 jit_extension->func_info.num = 0;
563 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
564 | ZEND_FUNC_JIT_ON_PROF_REQUEST
565 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
566 | ZEND_FUNC_JIT_ON_HOT_TRACE;
567 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
568 ssa = &jit_extension->func_info.ssa;
569
570 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
571 do {
572 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
573 break;
574 }
575
576 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
577 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info);
578 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
579 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
580 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
581 }
582 }
583
584 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
585 break;
586 }
587
588 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
589 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
590 }
591 return ssa;
592 } while (0);
593 }
594
595 memset(ssa, 0, sizeof(zend_ssa));
596 ssa->cfg.blocks_count = 1;
597
598 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
599 zend_cfg cfg;
600 void *checkpoint = zend_arena_checkpoint(CG(arena));
601
602 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
603 ssa->cfg.flags = cfg.flags;
604 } else{
605 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
606 }
607
608 /* TODO: move this to zend_cfg.c ? */
609 if (!op_array->function_name) {
610 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
611 }
612
613 zend_arena_release(&CG(arena), checkpoint);
614 }
615
616 return ssa;
617 }
618
619 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
620 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
621
zend_jit_trace_op_len(const zend_op * opline)622 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
623 {
624 int len;
625
626 switch (opline->opcode) {
627 case ZEND_ASSIGN_DIM:
628 case ZEND_ASSIGN_OBJ:
629 case ZEND_ASSIGN_STATIC_PROP:
630 case ZEND_ASSIGN_DIM_OP:
631 case ZEND_ASSIGN_OBJ_OP:
632 case ZEND_ASSIGN_STATIC_PROP_OP:
633 case ZEND_ASSIGN_OBJ_REF:
634 case ZEND_ASSIGN_STATIC_PROP_REF:
635 return 2; /* OP_DATA */
636 case ZEND_RECV_INIT:
637 len = 1;
638 opline++;
639 while (opline->opcode == ZEND_RECV_INIT) {
640 len++;
641 opline++;
642 }
643 return len;
644 case ZEND_BIND_GLOBAL:
645 len = 1;
646 opline++;
647 while (opline->opcode == ZEND_BIND_GLOBAL) {
648 len++;
649 opline++;
650 }
651 return len;
652 // case ZEND_IS_IDENTICAL:
653 // case ZEND_IS_NOT_IDENTICAL:
654 // case ZEND_IS_EQUAL:
655 // case ZEND_IS_NOT_EQUAL:
656 // case ZEND_IS_SMALLER:
657 // case ZEND_IS_SMALLER_OR_EQUAL:
658 // case ZEND_CASE:
659 // case ZEND_ISSET_ISEMPTY_CV:
660 // case ZEND_ISSET_ISEMPTY_VAR:
661 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
662 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
663 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
664 // case ZEND_INSTANCEOF:
665 // case ZEND_TYPE_CHECK:
666 // case ZEND_DEFINED:
667 // case ZEND_IN_ARRAY:
668 // case ZEND_ARRAY_KEY_EXISTS:
669 default:
670 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
671 return 2; /* JMPZ/JMPNZ */
672 }
673 return 1;
674 }
675 }
676
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)677 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
678 {
679 const zend_op_array *op_array;
680 zend_jit_trace_rec *p;
681 int k, vars_count;
682 zend_bitset use, def;
683 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
684 uint32_t set_size;
685 zend_ssa_phi *prev = NULL;
686 int level = 0;
687 ALLOCA_FLAG(use_heap);
688
689 op_array = trace_buffer->op_array;
690 set_size = zend_bitset_len(op_array->last_var + op_array->T);
691 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
692 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
693 def = use + set_size;
694 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
695 for (;;p++) {
696 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
697 const zend_op *opline = p->opline;
698 int len;
699
700 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
701 len = zend_jit_trace_op_len(opline);
702 while (len > 1) {
703 opline++;
704 if (opline->opcode != ZEND_OP_DATA) {
705 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
706 }
707 len--;
708 }
709 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
710 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
711 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
712 level++;
713 } else if (p->op == ZEND_JIT_TRACE_BACK) {
714 if (level == 0) {
715 // Phi for recursive calls and returns are not supported yet ???
716 assert(0);
717 } else {
718 level--;
719 }
720 } else if (p->op == ZEND_JIT_TRACE_END) {
721 break;
722 }
723 }
724
725 zend_bitset_intersection(use, def, set_size);
726
727 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
728 vars_count = op_array->last_var;
729 } else {
730 vars_count = op_array->last_var + op_array->T;
731 }
732 for (k = 0; k < vars_count; k++) {
733 if (zend_bitset_in(use, k)) {
734 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
735 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
736 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
737 sizeof(void*) * 2);
738 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
739 phi->sources[0] = STACK_VAR(stack, k);
740 phi->sources[1] = -1;
741 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
742 phi->pi = -1;
743 phi->var = k;
744 phi->ssa_var = ssa_vars_count;
745 SET_STACK_VAR(stack, k, ssa_vars_count);
746 ssa_vars_count++;
747 phi->block = 1;
748 if (prev) {
749 prev->next = phi;
750 } else {
751 tssa->blocks[1].phis = phi;
752 }
753 prev = phi;
754 }
755 }
756
757 free_alloca(use, use_heap);
758
759 return ssa_vars_count;
760 }
761
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)762 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
763 {
764 zend_ssa_phi *prev = NULL;
765 const zend_op_array *op_array = trace_buffer->op_array;
766 const zend_op *opline = trace_buffer[1].opline;
767 int count = opline - op_array->opcodes;
768 int i;
769
770 for(i = 0; i < count; i++) {
771 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
772 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
773 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
774 sizeof(void*) * 2);
775 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
776 phi->sources[0] = STACK_VAR(stack, i);
777 phi->sources[1] = -1;
778 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
779 phi->pi = -1;
780 phi->var = i;
781 phi->ssa_var = ssa_vars_count;
782 SET_STACK_VAR(stack, i, ssa_vars_count);
783 ssa_vars_count++;
784 phi->block = 1;
785 if (prev) {
786 prev->next = phi;
787 } else {
788 tssa->blocks[1].phis = phi;
789 }
790 prev = phi;
791 }
792 return ssa_vars_count;
793 }
794
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)795 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
796 {
797 const zend_op *opline = trace_buffer[1].opline - 1;
798 int i;
799
800 if (RETURN_VALUE_USED(opline)) {
801 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
802 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
803 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
804 sizeof(void*) * 2);
805
806 i = EX_VAR_TO_NUM(opline->result.var);
807 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
808 phi->sources[0] = STACK_VAR(stack, i);
809 phi->sources[1] = -1;
810 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
811 phi->pi = -1;
812 phi->var = i;
813 phi->ssa_var = ssa_vars_count;
814 SET_STACK_VAR(stack, i, ssa_vars_count);
815 ssa_vars_count++;
816 phi->block = 1;
817 tssa->blocks[1].phis = phi;
818 }
819 return ssa_vars_count;
820 }
821
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)822 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
823 {
824 int var, use;
825 zend_ssa_op *op;
826 zend_ssa_var_info *info;
827 unsigned int no_val;
828 zend_ssa_alias_kind alias;
829
830 if (tssa->vars[ssa_var].phi_use_chain) {
831 // TODO: this may be incorrect ???
832 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
833 } else {
834 var = ssa_var;
835 }
836 use = tssa->vars[var].use_chain;
837 if (use >= 0) {
838 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
839 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
840 if (tssa->ops[use].op1_use == var) {
841 no_val = ssa->vars[op->op1_use].no_val;
842 alias = ssa->vars[op->op1_use].alias;
843 info = ssa->var_info + op->op1_use;
844 } else if (tssa->ops[use].op2_use == var) {
845 no_val = ssa->vars[op->op2_use].no_val;
846 alias = ssa->vars[op->op2_use].alias;
847 info = ssa->var_info + op->op2_use;
848 } else if (tssa->ops[use].result_use == var) {
849 no_val = ssa->vars[op->result_use].no_val;
850 alias = ssa->vars[op->result_use].alias;
851 info = ssa->var_info + op->result_use;
852 } else {
853 assert(0);
854 return 0;
855 }
856 tssa->vars[ssa_var].no_val = no_val;
857 tssa->vars[ssa_var].alias = alias;
858 memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info));
859 return 1;
860 }
861 return 0;
862 }
863
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)864 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
865 {
866 zend_ssa_range tmp;
867 int def = tssa->vars[ssa_var].definition;
868
869 if (tssa->vars[ssa_var].alias == NO_ALIAS
870 && zend_inference_propagate_range(op_array, tssa, tssa_opcodes[def], &tssa->ops[def], ssa_var, &tmp)) {
871 tssa->var_info[ssa_var].range.min = tmp.min;
872 tssa->var_info[ssa_var].range.max = tmp.max;
873 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
874 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
875 tssa->var_info[ssa_var].has_range = 1;
876 }
877 }
878
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)879 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
880 {
881 int def;
882 zend_ssa_op *op;
883 zend_ssa_var_info *info;
884 unsigned int no_val;
885 zend_ssa_alias_kind alias;
886
887 def = tssa->vars[ssa_var].definition;
888 if (def >= 0) {
889 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
890 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
891 if (tssa->ops[def].op1_def == ssa_var) {
892 no_val = ssa->vars[op->op1_def].no_val;
893 alias = ssa->vars[op->op1_def].alias;
894 info = ssa->var_info + op->op1_def;
895 } else if (tssa->ops[def].op2_def == ssa_var) {
896 no_val = ssa->vars[op->op2_def].no_val;
897 alias = ssa->vars[op->op2_def].alias;
898 info = ssa->var_info + op->op2_def;
899 } else if (tssa->ops[def].result_def == ssa_var) {
900 no_val = ssa->vars[op->result_def].no_val;
901 alias = ssa->vars[op->result_def].alias;
902 info = ssa->var_info + op->result_def;
903 } else {
904 assert(0);
905 return;
906 }
907
908 tssa->vars[ssa_var].no_val = no_val;
909 tssa->vars[ssa_var].alias = alias;
910
911 if (!(info->type & MAY_BE_REF)) {
912 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
913 }
914
915 if (info->has_range) {
916 if (tssa->var_info[ssa_var].has_range) {
917 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
918 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
919 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
920 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
921 } else {
922 tssa->var_info[ssa_var].has_range = 1;
923 tssa->var_info[ssa_var].range = info->range;
924 }
925 }
926 }
927 }
928
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)929 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
930 {
931 int def;
932 zend_ssa_op *op;
933 zend_ssa_var_info *info;
934
935 def = tssa->vars[ssa_var].definition;
936 if (def >= 0) {
937 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
938 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
939 if (tssa->ops[def].op1_def == ssa_var) {
940 info = ssa->var_info + op->op1_def;
941 } else if (tssa->ops[def].op2_def == ssa_var) {
942 info = ssa->var_info + op->op2_def;
943 } else if (tssa->ops[def].result_def == ssa_var) {
944 info = ssa->var_info + op->result_def;
945 } else {
946 assert(0);
947 return 0;
948 }
949 tssa->var_info[ssa_var].type &= info->type;
950 if (info->ce) {
951 if (tssa->var_info[ssa_var].ce) {
952 if (tssa->var_info[ssa_var].ce != info->ce) {
953 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
954 /* everything fine */
955 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
956 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
957 } else {
958 // TODO: classes may implement the same interface ???
959 //ZEND_UNREACHABLE();
960 }
961 }
962 tssa->var_info[ssa_var].is_instanceof =
963 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
964 } else {
965 tssa->var_info[ssa_var].ce = info->ce;
966 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
967 }
968 }
969 if (info->has_range) {
970 if (tssa->var_info[ssa_var].has_range) {
971 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
972 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
973 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
974 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
975 } else {
976 tssa->var_info[ssa_var].has_range = 1;
977 tssa->var_info[ssa_var].range = info->range;
978 }
979 }
980 return 1;
981 }
982 return 0;
983 }
984
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)985 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
986 {
987 while (1) {
988 if (p->op == ZEND_JIT_TRACE_VM) {
989 if (p->opline->opcode == ZEND_DO_UCALL
990 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
991 || p->opline->opcode == ZEND_DO_FCALL) {
992 if (p->opline->result_type != IS_UNUSED) {
993 return ssa_op->result_def;
994 }
995 }
996 return -1;
997 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
998 /*skip */
999 } else {
1000 return -1;
1001 }
1002 p--;
1003 }
1004 }
1005
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1006 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1007 {
1008 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1009 p--;
1010 while (1) {
1011 if (p->op == ZEND_JIT_TRACE_VM) {
1012 if (p->opline->opcode == ZEND_INIT_FCALL
1013 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1014 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1015 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1016 || p->opline->opcode == ZEND_INIT_USER_CALL
1017 || p->opline->opcode == ZEND_NEW
1018 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1019 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
1020 return p->opline;
1021 }
1022 return NULL;
1023 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1024 /*skip */
1025 } else {
1026 return NULL;
1027 }
1028 p--;
1029 }
1030 } else {
1031 const zend_op *opline = NULL;
1032 int call_level = 0;
1033
1034 p++;
1035 while (1) {
1036 if (p->op == ZEND_JIT_TRACE_VM) {
1037 opline = p->opline;
1038 break;
1039 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1040 call_level++;
1041 /*skip */
1042 } else {
1043 return NULL;
1044 }
1045 p--;
1046 }
1047 if (opline) {
1048 while (opline > op_array->opcodes) {
1049 opline--;
1050 switch (opline->opcode) {
1051 case ZEND_INIT_FCALL:
1052 case ZEND_INIT_FCALL_BY_NAME:
1053 case ZEND_INIT_NS_FCALL_BY_NAME:
1054 case ZEND_INIT_METHOD_CALL:
1055 case ZEND_INIT_DYNAMIC_CALL:
1056 case ZEND_INIT_STATIC_METHOD_CALL:
1057 case ZEND_INIT_USER_CALL:
1058 case ZEND_NEW:
1059 if (call_level == 0) {
1060 return opline;
1061 }
1062 call_level--;
1063 break;
1064 case ZEND_DO_FCALL:
1065 case ZEND_DO_ICALL:
1066 case ZEND_DO_UCALL:
1067 case ZEND_DO_FCALL_BY_NAME:
1068 case ZEND_CALLABLE_CONVERT:
1069 call_level++;
1070 break;
1071 }
1072 }
1073 }
1074 }
1075 return NULL;
1076 }
1077
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1078 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1079 {
1080 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1081 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1082 int idx = tssa->vars[var].definition;
1083
1084 if (idx >= 0) {
1085 if (tssa->ops[idx].op1_def == var) {
1086 const zend_op *opline = ssa_opcodes[idx];
1087 if (opline->opcode == ZEND_PRE_DEC
1088 || opline->opcode == ZEND_PRE_INC
1089 || opline->opcode == ZEND_POST_DEC
1090 || opline->opcode == ZEND_POST_INC) {
1091 if (tssa->ops[idx].op1_use >= 0
1092 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1093 return 0;
1094 }
1095 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1096 return 0;
1097 }
1098 return 1;
1099 } else if (opline->opcode == ZEND_ASSIGN_OP
1100 && (opline->extended_value == ZEND_ADD
1101 || opline->extended_value == ZEND_SUB
1102 || opline->extended_value == ZEND_MUL)) {
1103 if ((opline->op2_type & (IS_VAR|IS_CV))
1104 && tssa->ops[idx].op2_use >= 0
1105 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1106 return 0;
1107 }
1108 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1109 return 0;
1110 }
1111 if (opline->op2_type == IS_CONST) {
1112 zval *zv = RT_CONSTANT(opline, opline->op2);
1113 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1114 return 0;
1115 }
1116 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1117 return 0;
1118 }
1119 return 1;
1120 }
1121 }
1122 if (tssa->ops[idx].result_def == var) {
1123 const zend_op *opline = ssa_opcodes[idx];
1124 if (opline->opcode == ZEND_ADD
1125 || opline->opcode == ZEND_SUB
1126 || opline->opcode == ZEND_MUL) {
1127 if ((opline->op1_type & (IS_VAR|IS_CV))
1128 && tssa->ops[idx].op1_use >= 0
1129 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1130 return 0;
1131 }
1132 if ((opline->op2_type & (IS_VAR|IS_CV))
1133 && tssa->ops[idx].op2_use >= 0
1134 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1135 return 0;
1136 }
1137 if (opline->op1_type == IS_CONST) {
1138 zval *zv = RT_CONSTANT(opline, opline->op1);
1139 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1140 return 0;
1141 }
1142 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1143 return 0;
1144 }
1145 if (opline->op2_type == IS_CONST) {
1146 zval *zv = RT_CONSTANT(opline, opline->op2);
1147 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1148 return 0;
1149 }
1150 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1151 return 0;
1152 }
1153 return 1;
1154 } else if (opline->opcode == ZEND_PRE_DEC
1155 || opline->opcode == ZEND_PRE_INC
1156 || opline->opcode == ZEND_POST_DEC
1157 || opline->opcode == ZEND_POST_INC) {
1158 if ((opline->op1_type & (IS_VAR|IS_CV))
1159 && tssa->ops[idx].op1_use >= 0
1160 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1161 return 0;
1162 }
1163 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1164 return 0;
1165 }
1166 return 1;
1167 }
1168 }
1169 }
1170 }
1171 return 0;
1172 }
1173
1174 typedef struct _zend_tssa {
1175 zend_ssa ssa;
1176 const zend_op **tssa_opcodes;
1177 int used_stack;
1178 } zend_tssa;
1179
1180 static const zend_op _nop_opcode = {0};
1181
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1182 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1183 {
1184 zend_ssa *tssa;
1185 zend_ssa_op *ssa_ops, *op;
1186 zend_ssa_var *ssa_vars;
1187 zend_ssa_var_info *ssa_var_info;
1188 const zend_op_array *op_array;
1189 const zend_op *opline;
1190 const zend_op **ssa_opcodes;
1191 zend_jit_trace_rec *p;
1192 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1193 zend_jit_trace_stack *stack;
1194 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1195 uint32_t optimization_level = 0;
1196 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1197 size_t frame_size, stack_top, stack_size, stack_bottom;
1198 zend_jit_op_array_trace_extension *jit_extension;
1199 zend_ssa *ssa;
1200 zend_jit_trace_stack_frame *frame, *top, *call;
1201 zend_ssa_var_info return_value_info;
1202
1203 /* 1. Count number of TSSA opcodes;
1204 * Count number of activation frames;
1205 * Calculate size of abstract stack;
1206 * Construct regular SSA for involved op_array */
1207 op_array = trace_buffer->op_array;
1208 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1209 stack_bottom = 0;
1210 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1211 ssa_ops_count = 0;
1212 call_level = 0;
1213 level = 0;
1214 num_op_arrays = 0;
1215 /* Remember op_array to cleanup */
1216 op_arrays[num_op_arrays++] = op_array;
1217 /* Build SSA */
1218 ssa = zend_jit_trace_build_ssa(op_array, script);
1219 for (;;p++) {
1220 if (p->op == ZEND_JIT_TRACE_VM) {
1221 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1222 const zend_op *opline = p->opline;
1223
1224 switch (opline->opcode) {
1225 case ZEND_INCLUDE_OR_EVAL:
1226 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1227 break;
1228 case ZEND_FETCH_R:
1229 case ZEND_FETCH_W:
1230 case ZEND_FETCH_RW:
1231 case ZEND_FETCH_FUNC_ARG:
1232 case ZEND_FETCH_IS:
1233 case ZEND_FETCH_UNSET:
1234 case ZEND_UNSET_VAR:
1235 case ZEND_ISSET_ISEMPTY_VAR:
1236 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1237 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1238 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1239 !op_array->function_name) {
1240 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1241 }
1242 break;
1243 }
1244 }
1245 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1246 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1247 call_level++;
1248 stack_top += zend_jit_trace_frame_size(p->op_array);
1249 if (stack_top > stack_size) {
1250 stack_size = stack_top;
1251 }
1252 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1253 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1254 if (p->func
1255 && p->func != (zend_function*)&zend_pass_function
1256 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1257 || zend_string_equals_literal(p->func->common.function_name, "compact")
1258 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1259 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1260 }
1261 }
1262 frame_size = zend_jit_trace_frame_size(p->op_array);
1263 if (call_level == 0) {
1264 if (stack_top + frame_size > stack_size) {
1265 stack_size = stack_top + frame_size;
1266 }
1267 } else {
1268 call_level--;
1269 stack_top -= frame_size;
1270 }
1271 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1272 op_array = p->op_array;
1273 if (call_level == 0) {
1274 stack_top += zend_jit_trace_frame_size(op_array);
1275 if (stack_top > stack_size) {
1276 stack_size = stack_top;
1277 }
1278 } else {
1279 call_level--;
1280 }
1281 level++;
1282 jit_extension =
1283 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1284 ssa = &jit_extension->func_info.ssa;
1285 if (ssa->cfg.blocks_count) {
1286 /* pass */
1287 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1288 /* Too many functions in single trace */
1289 *num_op_arrays_ptr = num_op_arrays;
1290 return NULL;
1291 } else {
1292 /* Remember op_array to cleanup */
1293 op_arrays[num_op_arrays++] = op_array;
1294 /* Build SSA */
1295 ssa = zend_jit_trace_build_ssa(op_array, script);
1296 }
1297 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1298 if (level == 0) {
1299 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1300 jit_extension =
1301 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1302 ssa = &jit_extension->func_info.ssa;
1303 if (ssa->cfg.blocks_count) {
1304 /* pass */
1305 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1306 /* Too many functions in single trace */
1307 *num_op_arrays_ptr = num_op_arrays;
1308 return NULL;
1309 } else {
1310 /* Remember op_array to cleanup */
1311 op_arrays[num_op_arrays++] = op_array;
1312 /* Build SSA */
1313 ssa = zend_jit_trace_build_ssa(op_array, script);
1314 }
1315 } else {
1316 stack_top -= zend_jit_trace_frame_size(op_array);
1317 level--;
1318 }
1319 op_array = p->op_array;
1320 } else if (p->op == ZEND_JIT_TRACE_END) {
1321 break;
1322 }
1323 }
1324 *num_op_arrays_ptr = num_op_arrays;
1325
1326 /* Allocate space for abstract stack */
1327 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1328
1329 /* 2. Construct TSSA */
1330 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1331 tssa->cfg.flags = ZEND_SSA_TSSA;
1332 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1333 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1334 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1335
1336 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1337 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1338 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1339 tssa->cfg.blocks_count = 2;
1340 tssa->cfg.edges_count = 2;
1341
1342 tssa->cfg.predecessors[0] = 0;
1343 tssa->cfg.predecessors[1] = 1;
1344
1345 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1346 tssa->cfg.blocks[0].successors_count = 1;
1347 tssa->cfg.blocks[0].predecessors_count = 0;
1348 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1349 tssa->cfg.blocks[0].successors[0] = 1;
1350
1351 tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1352 tssa->cfg.blocks[1].successors_count = 1;
1353 tssa->cfg.blocks[1].predecessors_count = 2;
1354 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1355 tssa->cfg.blocks[1].successors[1] = 1;
1356 } else {
1357 tssa->cfg.blocks_count = 1;
1358 tssa->cfg.edges_count = 0;
1359
1360 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1361 tssa->cfg.blocks[0].successors_count = 0;
1362 tssa->cfg.blocks[0].predecessors_count = 0;
1363 }
1364 ((zend_tssa*)tssa)->used_stack = -1;
1365
1366 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1367 return tssa;
1368 }
1369
1370 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1371 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1372 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1373 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1374 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1375
1376 op_array = trace_buffer->op_array;
1377 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1378 ssa_vars_count = op_array->last_var;
1379 } else {
1380 ssa_vars_count = op_array->last_var + op_array->T;
1381 }
1382 stack = frame->stack;
1383 for (i = 0; i < ssa_vars_count; i++) {
1384 SET_STACK_VAR(stack, i, i);
1385 }
1386
1387 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1388 // TODO: For tracing, it's possible, to create pseudo Phi functions
1389 // at the end of loop, without this additional pass (like LuaJIT) ???
1390 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1391 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1392 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1393 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1394 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1395 }
1396
1397 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1398 idx = 0;
1399 level = 0;
1400 for (;;p++) {
1401 if (p->op == ZEND_JIT_TRACE_VM) {
1402 opline = p->opline;
1403 ssa_opcodes[idx] = opline;
1404 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1405 idx++;
1406 len = zend_jit_trace_op_len(p->opline);
1407 while (len > 1) {
1408 opline++;
1409 ssa_opcodes[idx] = opline;
1410 if (opline->opcode != ZEND_OP_DATA) {
1411 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1412 }
1413 idx++;
1414 len--;
1415 }
1416 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1417 frame = zend_jit_trace_call_frame(frame, op_array);
1418 stack = frame->stack;
1419 op_array = p->op_array;
1420 level++;
1421 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1422 return NULL;
1423 }
1424 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1425 for (i = 0; i < op_array->last_var; i++) {
1426 SET_STACK_VAR(stack, i, ssa_vars_count++);
1427 }
1428 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1429 op_array = p->op_array;
1430 frame = zend_jit_trace_ret_frame(frame, op_array);
1431 stack = frame->stack;
1432 if (level == 0) {
1433 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1434 return NULL;
1435 }
1436 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1437 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1438 SET_STACK_VAR(stack, i, ssa_vars_count++);
1439 }
1440 } else {
1441 level--;
1442 }
1443 } else if (p->op == ZEND_JIT_TRACE_END) {
1444 break;
1445 }
1446 }
1447
1448 op_array = trace_buffer->op_array;
1449 tssa->vars_count = ssa_vars_count;
1450 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1451 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1452 vars_count = op_array->last_var;
1453 } else {
1454 vars_count = op_array->last_var + op_array->T;
1455 }
1456 i = 0;
1457 while (i < vars_count) {
1458 ssa_vars[i].var = i;
1459 ssa_vars[i].scc = -1;
1460 ssa_vars[i].definition = -1;
1461 ssa_vars[i].use_chain = -1;
1462 i++;
1463 }
1464 while (i < tssa->vars_count) {
1465 ssa_vars[i].var = -1;
1466 ssa_vars[i].scc = -1;
1467 ssa_vars[i].definition = -1;
1468 ssa_vars[i].use_chain = -1;
1469 i++;
1470 }
1471
1472 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1473 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1474 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1475 /* Update Phi sources */
1476 zend_ssa_phi *phi = tssa->blocks[1].phis;
1477
1478 while (phi) {
1479 phi->sources[1] = STACK_VAR(stack, phi->var);
1480 ssa_vars[phi->ssa_var].var = phi->var;
1481 ssa_vars[phi->ssa_var].definition_phi = phi;
1482 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1483 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1484 phi = phi->next;
1485 }
1486 }
1487
1488 /* 3. Compute use-def chains */
1489 idx = (ssa_ops_count - 1);
1490 op = ssa_ops + idx;
1491 while (idx >= 0) {
1492 opline = ssa_opcodes[idx];
1493 if (op->op1_use >= 0) {
1494 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1495 ssa_vars[op->op1_use].use_chain = idx;
1496 }
1497 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1498 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1499 ssa_vars[op->op2_use].use_chain = idx;
1500 }
1501 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1502 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1503 ssa_vars[op->result_use].use_chain = idx;
1504 }
1505 if (op->op1_def >= 0) {
1506 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1507 ssa_vars[op->op1_def].definition = idx;
1508 }
1509 if (op->op2_def >= 0) {
1510 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1511 ssa_vars[op->op2_def].definition = idx;
1512 }
1513 if (op->result_def >= 0) {
1514 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1515 ssa_vars[op->result_def].definition = idx;
1516 }
1517 op--;
1518 idx--;
1519 }
1520
1521 /* 4. Type inference */
1522 op_array = trace_buffer->op_array;
1523 jit_extension =
1524 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1525 ssa = &jit_extension->func_info.ssa;
1526
1527 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1528
1529 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1530 i = 0;
1531 while (i < op_array->last_var) {
1532 if (i < op_array->num_args) {
1533 if (ssa->var_info
1534 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1535 /* pass */
1536 } else {
1537 if (ssa->vars) {
1538 ssa_vars[i].no_val = ssa->vars[i].no_val;
1539 ssa_vars[i].alias = ssa->vars[i].alias;
1540 } else {
1541 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1542 }
1543 if (op_array->arg_info && i < trace_buffer[1].opline - op_array->opcodes) {
1544 zend_arg_info *arg_info = &op_array->arg_info[i];
1545 zend_class_entry *ce;
1546 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1547
1548 if (ZEND_ARG_SEND_MODE(arg_info)) {
1549 tmp |= MAY_BE_REF;
1550 }
1551 ssa_var_info[i].type = tmp;
1552 ssa_var_info[i].ce = ce;
1553 ssa_var_info[i].is_instanceof = 1;
1554 } else {
1555 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1556 }
1557 }
1558 } else {
1559 if (ssa->vars) {
1560 ssa_vars[i].no_val = ssa->vars[i].no_val;
1561 ssa_vars[i].alias = ssa->vars[i].alias;
1562 } else {
1563 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1564 }
1565 if (ssa_vars[i].alias == NO_ALIAS) {
1566 ssa_var_info[i].type = MAY_BE_UNDEF;
1567 } else {
1568 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1569 }
1570 }
1571 i++;
1572 }
1573 } else {
1574 int parent_vars_count = 0;
1575 zend_jit_trace_stack *parent_stack = NULL;
1576
1577 i = 0;
1578 if (parent_trace) {
1579 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1580 op_array->last_var + op_array->T);
1581 if (parent_vars_count) {
1582 parent_stack =
1583 zend_jit_traces[parent_trace].stack_map +
1584 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1585 }
1586 }
1587 while (i < op_array->last_var + op_array->T) {
1588 if (!ssa->var_info
1589 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1590 if (ssa->vars && i < ssa->vars_count) {
1591 ssa_vars[i].alias = ssa->vars[i].alias;
1592 } else {
1593 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1594 }
1595 if (i < op_array->last_var) {
1596 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1597 } else {
1598 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1599 }
1600 }
1601 if (i < parent_vars_count) {
1602 /* Initialize TSSA variable from parent trace */
1603 uint8_t op_type = STACK_TYPE(parent_stack, i);
1604
1605 if (op_type != IS_UNKNOWN) {
1606 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1607 if (!ssa_var_info[i].type
1608 && op_type == IS_UNDEF
1609 && i >= op_array->last_var) {
1610 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1611 ssa_var_info[i].type |= MAY_BE_UNDEF;
1612 }
1613 }
1614 }
1615 i++;
1616 }
1617 }
1618
1619 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1620 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1621 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1622 /* Propagate initial value through Phi functions */
1623 zend_ssa_phi *phi = tssa->blocks[1].phis;
1624
1625 while (phi) {
1626 if (!ssa->var_info
1627 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) {
1628 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1629 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1630 }
1631 phi = phi->next;
1632 }
1633 }
1634
1635 frame = JIT_G(current_frame);
1636 top = zend_jit_trace_call_frame(frame, op_array);
1637 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1638 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1639 frame->used_stack = 0;
1640 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1641 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
1642 }
1643 memset(&return_value_info, 0, sizeof(return_value_info));
1644
1645 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1646 max_used_stack = used_stack = 0;
1647 } else {
1648 max_used_stack = used_stack = -1;
1649 }
1650
1651 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1652 idx = 0;
1653 level = 0;
1654 opline = NULL;
1655 for (;;p++) {
1656 if (p->op == ZEND_JIT_TRACE_VM) {
1657 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1658 uint8_t val_type = IS_UNKNOWN;
1659 // zend_class_entry *op1_ce = NULL;
1660 zend_class_entry *op2_ce = NULL;
1661
1662 opline = p->opline;
1663
1664 op1_type = orig_op1_type = p->op1_type;
1665 op2_type = orig_op2_type = p->op2_type;
1666 op3_type = p->op3_type;
1667 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1668 op1_type = IS_UNKNOWN;
1669 }
1670 if (op1_type != IS_UNKNOWN) {
1671 op1_type &= ~IS_TRACE_PACKED;
1672 }
1673 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1674 op2_type = IS_UNKNOWN;
1675 }
1676 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1677 op3_type = IS_UNKNOWN;
1678 }
1679
1680 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1681 // op1_ce = (zend_class_entry*)(p+1)->ce;
1682 p++;
1683 }
1684 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1685 op2_ce = (zend_class_entry*)(p+1)->ce;
1686 p++;
1687 }
1688 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1689 val_type = (p+1)->op1_type;
1690 p++;
1691 }
1692
1693 switch (opline->opcode) {
1694 case ZEND_ASSIGN_OP:
1695 if (opline->extended_value == ZEND_POW
1696 || opline->extended_value == ZEND_DIV) {
1697 // TODO: check for division by zero ???
1698 break;
1699 }
1700 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1701 break;
1702 }
1703 ADD_OP1_TRACE_GUARD();
1704 ADD_OP2_TRACE_GUARD();
1705 break;
1706 case ZEND_ASSIGN_DIM_OP:
1707 if (opline->extended_value == ZEND_POW
1708 || opline->extended_value == ZEND_DIV) {
1709 // TODO: check for division by zero ???
1710 break;
1711 }
1712 if (opline->result_type != IS_UNUSED) {
1713 break;
1714 }
1715 if (op3_type != IS_UNKNOWN
1716 && !zend_jit_supported_binary_op(
1717 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1718 break;
1719 }
1720 ZEND_FALLTHROUGH;
1721 case ZEND_ASSIGN_DIM:
1722 if (opline->op1_type == IS_CV) {
1723 if ((opline+1)->op1_type == IS_CV
1724 && (opline+1)->op1.var == opline->op1.var) {
1725 /* skip $a[x] = $a; */
1726 break;
1727 }
1728 ADD_OP1_DATA_TRACE_GUARD();
1729 ADD_OP2_TRACE_GUARD();
1730 ADD_OP1_TRACE_GUARD();
1731 } else if (orig_op1_type != IS_UNKNOWN
1732 && (orig_op1_type & IS_TRACE_INDIRECT)
1733 && opline->result_type == IS_UNUSED) {
1734 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1735 ADD_OP1_DATA_TRACE_GUARD();
1736 }
1737 ADD_OP2_TRACE_GUARD();
1738 }
1739 if (op1_type == IS_ARRAY
1740 && ((opline->op2_type == IS_CONST
1741 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1742 || (opline->op2_type != IS_CONST
1743 && op2_type == IS_LONG))) {
1744
1745 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1746 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1747
1748 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1749 info->type |= MAY_BE_PACKED_GUARD;
1750 info->type &= ~MAY_BE_ARRAY_PACKED;
1751 }
1752 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1753 && val_type != IS_UNKNOWN
1754 && val_type != IS_UNDEF) {
1755 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1756
1757 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1758 info->type |= MAY_BE_PACKED_GUARD;
1759 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1760 }
1761 }
1762 }
1763 break;
1764 case ZEND_ASSIGN_OBJ_OP:
1765 if (opline->extended_value == ZEND_POW
1766 || opline->extended_value == ZEND_DIV) {
1767 // TODO: check for division by zero ???
1768 break;
1769 }
1770 if (opline->result_type != IS_UNUSED) {
1771 break;
1772 }
1773 ZEND_FALLTHROUGH;
1774 case ZEND_ASSIGN_OBJ:
1775 case ZEND_PRE_INC_OBJ:
1776 case ZEND_PRE_DEC_OBJ:
1777 case ZEND_POST_INC_OBJ:
1778 case ZEND_POST_DEC_OBJ:
1779 if (opline->op2_type != IS_CONST
1780 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1781 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1782 break;
1783 }
1784 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1785 if (opline->op1_type == IS_CV
1786 && (opline+1)->op1_type == IS_CV
1787 && (opline+1)->op1.var == opline->op1.var) {
1788 /* skip $a->prop += $a; */
1789 break;
1790 }
1791 ADD_OP1_DATA_TRACE_GUARD();
1792 }
1793 ADD_OP1_TRACE_GUARD();
1794 break;
1795 case ZEND_CONCAT:
1796 case ZEND_FAST_CONCAT:
1797 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1798 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1799 ADD_OP2_TRACE_GUARD();
1800 ADD_OP1_TRACE_GUARD();
1801 }
1802 break;
1803 case ZEND_IS_EQUAL:
1804 case ZEND_IS_NOT_EQUAL:
1805 case ZEND_IS_SMALLER:
1806 case ZEND_IS_SMALLER_OR_EQUAL:
1807 case ZEND_CASE:
1808 case ZEND_IS_IDENTICAL:
1809 case ZEND_IS_NOT_IDENTICAL:
1810 case ZEND_CASE_STRICT:
1811 case ZEND_BW_OR:
1812 case ZEND_BW_AND:
1813 case ZEND_BW_XOR:
1814 case ZEND_SL:
1815 case ZEND_SR:
1816 case ZEND_MOD:
1817 case ZEND_ADD:
1818 case ZEND_SUB:
1819 case ZEND_MUL:
1820 // case ZEND_DIV: // TODO: check for division by zero ???
1821 ADD_OP2_TRACE_GUARD();
1822 ZEND_FALLTHROUGH;
1823 case ZEND_ECHO:
1824 case ZEND_STRLEN:
1825 case ZEND_COUNT:
1826 case ZEND_QM_ASSIGN:
1827 case ZEND_FE_RESET_R:
1828 ADD_OP1_TRACE_GUARD();
1829 break;
1830 case ZEND_FE_FETCH_R:
1831 ADD_OP1_TRACE_GUARD();
1832 if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) {
1833
1834 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1835
1836 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1837 info->type |= MAY_BE_PACKED_GUARD;
1838 if (orig_op1_type & IS_TRACE_PACKED) {
1839 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1840 } else {
1841 info->type &= ~MAY_BE_ARRAY_PACKED;
1842 }
1843 }
1844 }
1845 break;
1846 case ZEND_VERIFY_RETURN_TYPE:
1847 if (opline->op1_type == IS_UNUSED) {
1848 /* Always throws */
1849 break;
1850 }
1851 if (opline->op1_type == IS_CONST) {
1852 /* TODO Different instruction format, has return value */
1853 break;
1854 }
1855 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1856 /* Not worth bothering with */
1857 break;
1858 }
1859 ADD_OP1_TRACE_GUARD();
1860 break;
1861 case ZEND_FETCH_DIM_FUNC_ARG:
1862 if (!frame
1863 || !frame->call
1864 || !frame->call->func
1865 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1866 break;
1867 }
1868 ADD_OP2_TRACE_GUARD();
1869 ADD_OP1_TRACE_GUARD();
1870 break;
1871 case ZEND_PRE_INC:
1872 case ZEND_PRE_DEC:
1873 case ZEND_POST_INC:
1874 case ZEND_POST_DEC:
1875 if (opline->op1_type != IS_CV) {
1876 break;
1877 }
1878 ADD_OP1_TRACE_GUARD();
1879 break;
1880 case ZEND_ASSIGN:
1881 if (opline->op1_type != IS_CV) {
1882 break;
1883 }
1884 ADD_OP2_TRACE_GUARD();
1885 if (op1_type != IS_UNKNOWN
1886 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1887 ADD_OP1_TRACE_GUARD();
1888 }
1889 break;
1890 case ZEND_CAST:
1891 if (opline->extended_value != op1_type) {
1892 break;
1893 }
1894 ADD_OP1_TRACE_GUARD();
1895 break;
1896 case ZEND_JMPZ:
1897 case ZEND_JMPNZ:
1898 case ZEND_JMPZ_EX:
1899 case ZEND_JMPNZ_EX:
1900 case ZEND_BOOL:
1901 case ZEND_BOOL_NOT:
1902 ADD_OP1_TRACE_GUARD();
1903 break;
1904 case ZEND_ISSET_ISEMPTY_CV:
1905 if ((opline->extended_value & ZEND_ISEMPTY)) {
1906 // TODO: support for empty() ???
1907 break;
1908 }
1909 ADD_OP1_TRACE_GUARD();
1910 break;
1911 case ZEND_IN_ARRAY:
1912 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
1913 break;
1914 }
1915 ADD_OP1_TRACE_GUARD();
1916 break;
1917 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
1918 if ((opline->extended_value & ZEND_ISEMPTY)) {
1919 // TODO: support for empty() ???
1920 break;
1921 }
1922 ZEND_FALLTHROUGH;
1923 case ZEND_FETCH_DIM_R:
1924 case ZEND_FETCH_DIM_IS:
1925 case ZEND_FETCH_LIST_R:
1926 ADD_OP1_TRACE_GUARD();
1927 ADD_OP2_TRACE_GUARD();
1928
1929 if (op1_type == IS_ARRAY
1930 && opline->op1_type != IS_CONST
1931 && ((opline->op2_type == IS_CONST
1932 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1933 || (opline->op2_type != IS_CONST
1934 && op2_type == IS_LONG))) {
1935
1936 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1937
1938 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1939 info->type |= MAY_BE_PACKED_GUARD;
1940 if (orig_op1_type & IS_TRACE_PACKED) {
1941 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1942 } else {
1943 info->type &= ~MAY_BE_ARRAY_PACKED;
1944 }
1945 }
1946 }
1947 break;
1948 case ZEND_FETCH_DIM_W:
1949 case ZEND_FETCH_DIM_RW:
1950 // case ZEND_FETCH_DIM_UNSET:
1951 case ZEND_FETCH_LIST_W:
1952 if (opline->op1_type != IS_CV
1953 && (orig_op1_type == IS_UNKNOWN
1954 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
1955 break;
1956 }
1957 ADD_OP1_TRACE_GUARD();
1958 ADD_OP2_TRACE_GUARD();
1959 if (op1_type == IS_ARRAY
1960 && !(orig_op1_type & IS_TRACE_PACKED)
1961 && ((opline->op2_type == IS_CONST
1962 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1963 || (opline->op2_type != IS_CONST
1964 && op2_type == IS_LONG))) {
1965
1966 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1967
1968 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1969 info->type |= MAY_BE_PACKED_GUARD;
1970 info->type &= ~MAY_BE_ARRAY_PACKED;
1971 }
1972 }
1973 break;
1974 case ZEND_SEND_VAL_EX:
1975 case ZEND_SEND_VAR_EX:
1976 case ZEND_SEND_VAR_NO_REF_EX:
1977 if (opline->op2_type == IS_CONST) {
1978 /* Named parameters not supported in JIT */
1979 break;
1980 }
1981 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
1982 goto propagate_arg;
1983 }
1984 ZEND_FALLTHROUGH;
1985 case ZEND_SEND_VAL:
1986 case ZEND_SEND_VAR:
1987 case ZEND_SEND_VAR_NO_REF:
1988 case ZEND_SEND_FUNC_ARG:
1989 if (opline->op2_type == IS_CONST) {
1990 /* Named parameters not supported in JIT */
1991 break;
1992 }
1993 ADD_OP1_TRACE_GUARD();
1994 propagate_arg:
1995 /* Propagate argument type */
1996 if (frame->call
1997 && frame->call->func
1998 && frame->call->func->type == ZEND_USER_FUNCTION
1999 && opline->op2.num <= frame->call->func->op_array.num_args) {
2000 uint32_t info;
2001
2002 if (opline->op1_type == IS_CONST) {
2003 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
2004 } else {
2005 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2006 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
2007 }
2008 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
2009 zend_arg_info *arg_info;
2010
2011 ZEND_ASSERT(frame->call->func->op_array.arg_info);
2012 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
2013 if (ZEND_TYPE_IS_SET(arg_info->type)) {
2014 zend_class_entry *ce;
2015 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2016 info &= tmp;
2017 if (!info) {
2018 break;
2019 }
2020 }
2021 }
2022 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2023 info |= MAY_BE_RCN;
2024 }
2025 if (info & MAY_BE_UNDEF) {
2026 info |= MAY_BE_NULL;
2027 info &= ~MAY_BE_UNDEF;
2028 }
2029 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2030 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2031 }
2032 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2033 }
2034 break;
2035 case ZEND_RETURN:
2036 ADD_OP1_TRACE_GUARD();
2037 /* Propagate return value types */
2038 if (opline->op1_type == IS_UNUSED) {
2039 return_value_info.type = MAY_BE_NULL;
2040 } else if (opline->op1_type == IS_CONST) {
2041 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2042 } else {
2043 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2044 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2045 if (return_value_info.type & MAY_BE_UNDEF) {
2046 return_value_info.type &= ~MAY_BE_UNDEF;
2047 return_value_info.type |= MAY_BE_NULL;
2048 }
2049 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2050 /* CVs are going to be destructed and the reference-counter
2051 of return value may be decremented to 1 */
2052 return_value_info.type |= MAY_BE_RC1;
2053 }
2054 return_value_info.type &= ~MAY_BE_GUARD;
2055 }
2056 break;
2057 case ZEND_CHECK_FUNC_ARG:
2058 if (!frame
2059 || !frame->call
2060 || !frame->call->func) {
2061 break;
2062 }
2063 if (opline->op2_type == IS_CONST
2064 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2065 /* Named parameters not supported in JIT */
2066 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2067 break;
2068 }
2069 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2070 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2071 } else {
2072 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2073 }
2074 break;
2075 case ZEND_FETCH_OBJ_FUNC_ARG:
2076 if (!frame
2077 || !frame->call
2078 || !frame->call->func
2079 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2080 break;
2081 }
2082 ZEND_FALLTHROUGH;
2083 case ZEND_FETCH_OBJ_R:
2084 case ZEND_FETCH_OBJ_IS:
2085 case ZEND_FETCH_OBJ_W:
2086 if (opline->op2_type != IS_CONST
2087 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2088 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2089 break;
2090 }
2091 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2092 ADD_OP1_TRACE_GUARD();
2093 }
2094 break;
2095 case ZEND_INIT_METHOD_CALL:
2096 if (opline->op2_type != IS_CONST
2097 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2098 break;
2099 }
2100 ADD_OP1_TRACE_GUARD();
2101 break;
2102 case ZEND_INIT_DYNAMIC_CALL:
2103 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2104 ADD_OP2_TRACE_GUARD();
2105 }
2106 break;
2107 case ZEND_SEND_ARRAY:
2108 case ZEND_SEND_UNPACK:
2109 case ZEND_CHECK_UNDEF_ARGS:
2110 case ZEND_INCLUDE_OR_EVAL:
2111 max_used_stack = used_stack = -1;
2112 break;
2113 case ZEND_TYPE_CHECK:
2114 if (opline->extended_value == MAY_BE_RESOURCE) {
2115 // TODO: support for is_resource() ???
2116 break;
2117 }
2118 if (op1_type != IS_UNKNOWN
2119 && (opline->extended_value == (1 << op1_type)
2120 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2121 /* add guards only for exact checks, to avoid code duplication */
2122 ADD_OP1_TRACE_GUARD();
2123 }
2124 break;
2125 case ZEND_ROPE_INIT:
2126 case ZEND_ROPE_ADD:
2127 case ZEND_ROPE_END:
2128 if (op2_type == IS_STRING) {
2129 ADD_OP2_TRACE_GUARD();
2130 }
2131 break;
2132 default:
2133 break;
2134 }
2135 len = zend_jit_trace_op_len(opline);
2136 if (ssa->var_info) {
2137 /* Add statically inferred ranges */
2138 if (ssa_ops[idx].op1_def >= 0) {
2139 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2140 }
2141 if (ssa_ops[idx].op2_def >= 0) {
2142 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2143 }
2144 if (ssa_ops[idx].result_def >= 0) {
2145 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2146 }
2147 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2148 if (ssa_ops[idx+1].op1_def >= 0) {
2149 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2150 }
2151 if (ssa_ops[idx+1].op2_def >= 0) {
2152 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2153 }
2154 if (ssa_ops[idx+1].result_def >= 0) {
2155 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2156 }
2157 }
2158 } else {
2159 if (ssa_ops[idx].op1_def >= 0) {
2160 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2161 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2162 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2163 }
2164 }
2165 if (ssa_ops[idx].op2_def >= 0) {
2166 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2167 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2168 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2169 }
2170 }
2171 if (ssa_ops[idx].result_def >= 0) {
2172 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2173 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2174 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2175 }
2176 }
2177 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2178 if (ssa_ops[idx+1].op1_def >= 0) {
2179 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2180 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2181 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2182 }
2183 }
2184 if (ssa_ops[idx+1].op2_def >= 0) {
2185 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2186 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2187 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2188 }
2189 }
2190 if (ssa_ops[idx+1].result_def >= 0) {
2191 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2192 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2193 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2194 }
2195 }
2196 }
2197 }
2198 if (opline->opcode == ZEND_RECV_INIT
2199 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2200 /* RECV_INIT always copy the constant */
2201 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2202 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2203 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2204 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2205 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2206 }
2207 } else {
2208 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2209 // TODO:
2210 assert(0);
2211 }
2212 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2213 && ssa_ops[idx].op1_def >= 0
2214 && op1_type == IS_ARRAY
2215 && (orig_op1_type & IS_TRACE_PACKED)
2216 && val_type != IS_UNKNOWN
2217 && val_type != IS_UNDEF
2218 && ((opline->op2_type == IS_CONST
2219 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2220 || (opline->op2_type != IS_CONST
2221 && op2_type == IS_LONG))) {
2222 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2223
2224 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2225 }
2226 }
2227 if (ssa->var_info) {
2228 /* Add statically inferred restrictions */
2229 if (ssa_ops[idx].op1_def >= 0) {
2230 if (opline->opcode == ZEND_SEND_VAR_EX
2231 && frame
2232 && frame->call
2233 && frame->call->func
2234 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2235 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2236 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2237 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2238 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2239 }
2240 } else {
2241 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2242 }
2243 }
2244 if (ssa_ops[idx].op2_def >= 0) {
2245 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2246 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2247 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2248 }
2249 }
2250 if (ssa_ops[idx].result_def >= 0) {
2251 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2252 }
2253 }
2254 idx++;
2255 while (len > 1) {
2256 opline++;
2257 if (opline->opcode != ZEND_OP_DATA) {
2258 if (ssa->var_info) {
2259 /* Add statically inferred ranges */
2260 if (ssa_ops[idx].op1_def >= 0) {
2261 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2262 }
2263 if (ssa_ops[idx].op2_def >= 0) {
2264 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2265 }
2266 if (ssa_ops[idx].result_def >= 0) {
2267 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2268 }
2269 } else {
2270 if (ssa_ops[idx].op1_def >= 0) {
2271 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2272 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2273 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2274 }
2275 }
2276 if (ssa_ops[idx].op2_def >= 0) {
2277 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2278 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2279 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2280 }
2281 }
2282 if (ssa_ops[idx].result_def >= 0) {
2283 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2284 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2285 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2286 }
2287 }
2288 }
2289 if (opline->opcode == ZEND_RECV_INIT
2290 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2291 /* RECV_INIT always copy the constant */
2292 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2293 } else {
2294 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2295 // TODO:
2296 assert(0);
2297 }
2298 }
2299 }
2300 if (ssa->var_info) {
2301 /* Add statically inferred restrictions */
2302 if (ssa_ops[idx].op1_def >= 0) {
2303 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2304 }
2305 if (ssa_ops[idx].op2_def >= 0) {
2306 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2307 }
2308 if (ssa_ops[idx].result_def >= 0) {
2309 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2310 }
2311 }
2312 idx++;
2313 len--;
2314 }
2315
2316 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2317 op_array = p->op_array;
2318 jit_extension =
2319 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2320 ssa = &jit_extension->func_info.ssa;
2321
2322 call = frame->call;
2323 if (!call) {
2324 /* Trace missed INIT_FCALL opcode */
2325 call = top;
2326 TRACE_FRAME_INIT(call, op_array, 0, 0);
2327 call->used_stack = 0;
2328 top = zend_jit_trace_call_frame(top, op_array);
2329 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2330 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
2331 }
2332 } else {
2333 ZEND_ASSERT(&call->func->op_array == op_array);
2334 }
2335 frame->call = call->prev;
2336 call->prev = frame;
2337 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2338 frame = call;
2339
2340 level++;
2341 i = 0;
2342 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2343 while (i < op_array->last_var) {
2344 ssa_vars[v].var = i;
2345 if (i < op_array->num_args) {
2346 if (ssa->var_info
2347 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2348 /* pass */
2349 } else {
2350 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2351 if (op_array->arg_info) {
2352 zend_arg_info *arg_info = &op_array->arg_info[i];
2353 zend_class_entry *ce;
2354 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2355
2356 if (ZEND_ARG_SEND_MODE(arg_info)) {
2357 tmp |= MAY_BE_REF;
2358 }
2359 ssa_var_info[v].type = tmp;
2360 ssa_var_info[v].ce = ce;
2361 ssa_var_info[v].is_instanceof = 1;
2362 } else {
2363 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2364 }
2365 }
2366 } else {
2367 if (ssa->vars) {
2368 ssa_vars[v].no_val = ssa->vars[i].no_val;
2369 ssa_vars[v].alias = ssa->vars[i].alias;
2370 } else {
2371 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2372 }
2373 if (ssa_vars[v].alias == NO_ALIAS) {
2374 ssa_var_info[v].type = MAY_BE_UNDEF;
2375 } else {
2376 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2377 }
2378 }
2379 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2380 && i < op_array->num_args) {
2381 /* Propagate argument type */
2382 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2383 }
2384 i++;
2385 v++;
2386 }
2387 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2388 op_array = p->op_array;
2389 jit_extension =
2390 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2391 ssa = &jit_extension->func_info.ssa;
2392 if (level == 0) {
2393 i = 0;
2394 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2395 while (i < op_array->last_var) {
2396 ssa_vars[v].var = i;
2397 if (!ssa->var_info
2398 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2399 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2400 }
2401 i++;
2402 v++;
2403 }
2404 while (i < op_array->last_var + op_array->T) {
2405 ssa_vars[v].var = i;
2406 if (!ssa->var_info
2407 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2408 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2409 }
2410 i++;
2411 v++;
2412 }
2413 if (return_value_info.type != 0) {
2414 zend_jit_trace_rec *q = p + 1;
2415 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2416 q++;
2417 }
2418 if (q->op == ZEND_JIT_TRACE_VM
2419 || (q->op == ZEND_JIT_TRACE_END
2420 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2421 const zend_op *opline = q->opline - 1;
2422 if (opline->result_type != IS_UNUSED) {
2423 ssa_var_info[
2424 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2425 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2426 }
2427 }
2428 memset(&return_value_info, 0, sizeof(return_value_info));
2429 }
2430 } else {
2431 level--;
2432 if (return_value_info.type != 0) {
2433 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2434 const zend_op *opline = (p+1)->opline - 1;
2435 if (opline->result_type != IS_UNUSED) {
2436 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2437 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2438 }
2439 }
2440 }
2441 memset(&return_value_info, 0, sizeof(return_value_info));
2442 }
2443 }
2444
2445 top = frame;
2446 if (frame->prev) {
2447 if (used_stack > 0) {
2448 used_stack -= frame->used_stack;
2449 }
2450 frame = frame->prev;
2451 ZEND_ASSERT(&frame->func->op_array == op_array);
2452 } else {
2453 max_used_stack = used_stack = -1;
2454 frame = zend_jit_trace_ret_frame(frame, op_array);
2455 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2456 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2457 frame->used_stack = 0;
2458 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2459 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
2460 }
2461 }
2462
2463 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2464 call = top;
2465 TRACE_FRAME_INIT(call, p->func, 0, 0);
2466 call->prev = frame->call;
2467 call->used_stack = 0;
2468 frame->call = call;
2469 top = zend_jit_trace_call_frame(top, p->op_array);
2470 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2471 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2472 SET_STACK_INFO(call->stack, i, -1);
2473 }
2474 }
2475 if (used_stack >= 0
2476 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2477 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2478 max_used_stack = used_stack = -1;
2479 } else {
2480 const zend_op *opline = (p-1)->opline;
2481
2482 switch (opline->opcode) {
2483 case ZEND_INIT_FCALL:
2484 case ZEND_INIT_FCALL_BY_NAME:
2485 case ZEND_INIT_NS_FCALL_BY_NAME:
2486 case ZEND_INIT_METHOD_CALL:
2487 case ZEND_INIT_DYNAMIC_CALL:
2488 //case ZEND_INIT_STATIC_METHOD_CALL:
2489 //case ZEND_INIT_USER_CALL:
2490 //case ZEND_NEW:
2491 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2492 used_stack += frame->used_stack;
2493 if (used_stack > max_used_stack) {
2494 max_used_stack = used_stack;
2495 }
2496 break;
2497 default:
2498 max_used_stack = used_stack = -1;
2499 }
2500 }
2501 }
2502 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2503 call = frame->call;
2504 if (call) {
2505 top = call;
2506 frame->call = call->prev;
2507 }
2508
2509 if (idx > 0
2510 && ssa_ops[idx-1].result_def >= 0
2511 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2512 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2513 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2514 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2515 opline->opcode == ZEND_DO_FCALL ||
2516 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2517
2518 if (opline->result_type != IS_UNDEF) {
2519 zend_class_entry *ce;
2520 const zend_function *func = p->func;
2521 zend_arg_info *ret_info = func->common.arg_info - 1;
2522 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2523
2524 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2525 }
2526 }
2527 } else if (p->op == ZEND_JIT_TRACE_END) {
2528 break;
2529 }
2530 }
2531
2532 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2533
2534 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2535 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2536 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2537 /* Propagate guards through Phi sources */
2538 zend_ssa_phi *phi = tssa->blocks[1].phis;
2539
2540 op_array = trace_buffer->op_array;
2541 jit_extension =
2542 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2543 ssa = &jit_extension->func_info.ssa;
2544
2545 while (phi) {
2546 uint32_t t = ssa_var_info[phi->ssa_var].type;
2547
2548 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2549 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2550 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2551
2552 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2553 if (!((t0 | t1) & MAY_BE_GUARD)) {
2554 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2555 }
2556 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2557 if (!(t1 & MAY_BE_GUARD)
2558 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2559 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2560 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2561 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2562 MAY_BE_GUARD;
2563 if (!(t0 & MAY_BE_ARRAY)) {
2564 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2565 }
2566 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2567 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2568 }
2569 ssa_var_info[phi->sources[0]].type = t0;
2570 ssa_var_info[phi->sources[0]].type = t0;
2571 }
2572 } else {
2573 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2574 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2575 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2576 MAY_BE_GUARD;
2577 if (!(t0 & MAY_BE_ARRAY)) {
2578 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2579 }
2580 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2581 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2582 }
2583 ssa_var_info[phi->sources[0]].type = t0;
2584 }
2585 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2586 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2587 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2588 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2589 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2590 MAY_BE_GUARD;
2591 if (!(t1 & MAY_BE_ARRAY)) {
2592 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2593 }
2594 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2595 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2596 }
2597 ssa_var_info[phi->sources[1]].type = t1;
2598 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2599 }
2600 }
2601 }
2602 t = ssa_var_info[phi->ssa_var].type;
2603 }
2604
2605 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2606 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2607 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2608
2609 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2610 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2611 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2612 }
2613 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2614 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2615 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2616 ssa_var_info[phi->sources[0]].type =
2617 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2618 }
2619 }
2620 }
2621 phi = phi->next;
2622 }
2623 }
2624
2625 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2626 if (parent_trace) {
2627 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2628 ZEND_JIT_TRACE_NUM,
2629 parent_trace,
2630 exit_num,
2631 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2632 trace_buffer->op_array->scope ? "::" : "",
2633 trace_buffer->op_array->function_name ?
2634 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2635 ZSTR_VAL(trace_buffer->op_array->filename),
2636 trace_buffer[1].opline->lineno);
2637 } else {
2638 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2639 ZEND_JIT_TRACE_NUM,
2640 zend_jit_trace_star_desc(trace_buffer->start),
2641 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2642 trace_buffer->op_array->scope ? "::" : "",
2643 trace_buffer->op_array->function_name ?
2644 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2645 ZSTR_VAL(trace_buffer->op_array->filename),
2646 trace_buffer[1].opline->lineno);
2647 }
2648 zend_jit_dump_trace(trace_buffer, tssa);
2649 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2650 uint32_t idx = trace_buffer[1].last;
2651 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2652 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2653 ZEND_JIT_TRACE_NUM,
2654 link_to);
2655 } else {
2656 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2657 ZEND_JIT_TRACE_NUM,
2658 zend_jit_trace_stop_description[trace_buffer->stop]);
2659 }
2660 }
2661
2662 return tssa;
2663 }
2664
zend_jit_close_var(zend_jit_trace_stack * stack,uint32_t n,int * start,int * end,uint8_t * flags,int line)2665 static void zend_jit_close_var(zend_jit_trace_stack *stack, uint32_t n, int *start, int *end, uint8_t *flags, int line)
2666 {
2667 int32_t var = STACK_VAR(stack, n);
2668
2669 if (var >= 0 && start[var] >= 0 && !(flags[var] & ZREG_LAST_USE)) {
2670 // TODO: shrink interval to last side exit ????
2671 end[var] = line;
2672 }
2673 }
2674
zend_jit_trace_use_var(int line,int var,int def,int use_chain,int * start,int * end,uint8_t * flags,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2675 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, int *start, int *end, uint8_t *flags, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2676 {
2677 ZEND_ASSERT(start[var] >= 0);
2678 ZEND_ASSERT(!(flags[var] & ZREG_LAST_USE));
2679 end[var] = line;
2680 if (def >= 0) {
2681 flags[var] |= ZREG_LAST_USE;
2682 } else if (use_chain < 0 && (flags[var] & (ZREG_LOAD|ZREG_STORE))) {
2683 flags[var] |= ZREG_LAST_USE;
2684 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2685 /* pass */
2686 } else if (op_array_ssa->vars) {
2687 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2688
2689 if (ssa->ops[line].op1_use == var) {
2690 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2691 flags[var] |= ZREG_LAST_USE;
2692 }
2693 } else if (ssa->ops[line].op2_use == var) {
2694 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2695 flags[var] |= ZREG_LAST_USE;
2696 }
2697 } else if (ssa->ops[line].result_use == var) {
2698 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2699 flags[var] |= ZREG_LAST_USE;
2700 }
2701 }
2702 }
2703 }
2704
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2705 static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2706 {
2707 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2708 zend_jit_trace_rec *p;
2709 const zend_op_array *op_array;
2710 zend_jit_op_array_trace_extension *jit_extension;
2711 const zend_ssa *op_array_ssa;
2712 const zend_ssa_op *ssa_op;
2713 int i, j, idx, count, level;
2714 int last_idx = -1;
2715 int *start, *end;
2716 uint8_t *flags;
2717 const zend_op_array **vars_op_array;
2718 zend_lifetime_interval **intervals, *list, *ival;
2719 void *checkpoint;
2720 zend_jit_trace_stack_frame *frame;
2721 zend_jit_trace_stack *stack;
2722 uint32_t parent_vars_count = parent_trace ?
2723 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2724 zend_jit_trace_stack *parent_stack = parent_trace ?
2725 zend_jit_traces[parent_trace].stack_map +
2726 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2727 ALLOCA_FLAG(use_heap);
2728
2729 ZEND_ASSERT(ssa->var_info != NULL);
2730
2731 start = do_alloca(sizeof(int) * ssa->vars_count * 2 +
2732 ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count),
2733 use_heap);
2734 if (!start) {
2735 return NULL;
2736 }
2737 end = start + ssa->vars_count;
2738 flags = (uint8_t*)(end + ssa->vars_count);
2739 checkpoint = zend_arena_checkpoint(CG(arena));
2740 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2741
2742 memset(start, -1, sizeof(int) * ssa->vars_count * 2);
2743 memset(flags, 0, sizeof(uint8_t) * ssa->vars_count);
2744 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2745
2746 op_array = trace_buffer->op_array;
2747 jit_extension =
2748 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2749 op_array_ssa = &jit_extension->func_info.ssa;
2750 frame = JIT_G(current_frame);
2751 frame->prev = NULL;
2752 frame->func = (const zend_function*)op_array;
2753 stack = frame->stack;
2754
2755 count = 0;
2756
2757 i = 0;
2758 j = op_array->last_var;
2759 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2760 j += op_array->T;
2761 }
2762 while (i < j) {
2763 SET_STACK_VAR(stack, i, i);
2764 vars_op_array[i] = op_array;
2765 /* We don't start intervals for variables used in Phi */
2766 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2767 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2768 && ssa->vars[i].alias == NO_ALIAS
2769 && zend_jit_var_supports_reg(ssa, i)) {
2770 start[i] = 0;
2771 if (i < parent_vars_count
2772 && STACK_REG(parent_stack, i) != ZREG_NONE
2773 && STACK_REG(parent_stack, i) < ZREG_NUM) {
2774 /* We will try to reuse register from parent trace */
2775 flags[i] = STACK_FLAGS(parent_stack, i);
2776 count += 2;
2777 } else {
2778 flags[i] = ZREG_LOAD;
2779 count++;
2780 }
2781 }
2782 i++;
2783 }
2784
2785 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2786 j = op_array->last_var + op_array->T;
2787 while (i < j) {
2788 SET_STACK_VAR(stack, i, -1);
2789 i++;
2790 }
2791 }
2792
2793 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2794 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2795 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2796 zend_ssa_phi *phi = ssa->blocks[1].phis;
2797
2798 while (phi) {
2799 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2800 vars_op_array[phi->ssa_var] = op_array;
2801 if (ssa->vars[phi->ssa_var].use_chain >= 0
2802 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2803 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2804 start[phi->ssa_var] = 0;
2805 count++;
2806 }
2807 phi = phi->next;
2808 }
2809 }
2810
2811 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2812 level = 0;
2813 ssa_op = ssa->ops;
2814 idx = 0;
2815 for (;;p++) {
2816 if (p->op == ZEND_JIT_TRACE_VM) {
2817 const zend_op *opline = p->opline;
2818 int len;
2819 bool support_opline;
2820
2821 support_opline =
2822 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2823
2824 if (support_opline
2825 && opline->opcode == ZEND_ASSIGN
2826 && opline->op1_type == IS_CV
2827 && ssa_op->op1_def >= 0
2828 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2829 /* avoid register allocation in case of possibility of indirect modification*/
2830 support_opline = 0;
2831 }
2832
2833 if (ssa_op->op1_use >= 0
2834 && start[ssa_op->op1_use] >= 0
2835 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2836 if (support_opline) {
2837 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2838 if (opline->op1_type != IS_CV) {
2839 if (opline->opcode == ZEND_CASE
2840 || opline->opcode == ZEND_CASE_STRICT
2841 || opline->opcode == ZEND_SWITCH_LONG
2842 || opline->opcode == ZEND_MATCH
2843 || opline->opcode == ZEND_FETCH_LIST_R
2844 || opline->opcode == ZEND_COPY_TMP
2845 || opline->opcode == ZEND_SWITCH_STRING
2846 || opline->opcode == ZEND_FE_FETCH_R
2847 || opline->opcode == ZEND_FE_FETCH_RW
2848 || opline->opcode == ZEND_FETCH_LIST_W
2849 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2850 || opline->opcode == ZEND_BIND_LEXICAL
2851 || opline->opcode == ZEND_ROPE_ADD) {
2852 /* The value is kept alive and may be used outside of the trace */
2853 flags[ssa_op->op1_use] |= ZREG_STORE;
2854 } else {
2855 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2856 }
2857 }
2858 } else {
2859 start[ssa_op->op1_use] = -1;
2860 end[ssa_op->op1_use] = -1;
2861 count--;
2862 }
2863 }
2864 if (ssa_op->op2_use >= 0
2865 && ssa_op->op2_use != ssa_op->op1_use
2866 && start[ssa_op->op2_use] >= 0
2867 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2868 if (support_opline) {
2869 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2870 if (opline->op2_type != IS_CV) {
2871 flags[ssa_op->op2_use] |= ZREG_LAST_USE;
2872 }
2873 } else {
2874 start[ssa_op->op2_use] = -1;
2875 end[ssa_op->op2_use] = -1;
2876 count--;
2877 }
2878 }
2879 if (ssa_op->result_use >= 0
2880 && ssa_op->result_use != ssa_op->op1_use
2881 && ssa_op->result_use != ssa_op->op2_use
2882 && start[ssa_op->result_use] >= 0
2883 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2884 if (support_opline) {
2885 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2886 } else {
2887 start[ssa_op->result_use] = -1;
2888 end[ssa_op->result_use] = -1;
2889 count--;
2890 }
2891 }
2892
2893 if (ssa_op->op1_def >= 0) {
2894 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2895 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2896 }
2897 if (ssa_op->op2_def >= 0) {
2898 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op2.var), start, end, flags, idx);
2899 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
2900 }
2901 if (ssa_op->result_def >= 0) {
2902 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2903 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2904 }
2905
2906 if (support_opline) {
2907 if (ssa_op->result_def >= 0
2908 && (ssa->vars[ssa_op->result_def].use_chain >= 0
2909 || ssa->vars[ssa_op->result_def].phi_use_chain)
2910 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
2911 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
2912 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
2913 || opline->opcode == ZEND_PRE_INC
2914 || opline->opcode == ZEND_PRE_DEC
2915 || opline->opcode == ZEND_POST_INC
2916 || opline->opcode == ZEND_POST_DEC
2917 || opline->opcode == ZEND_ADD
2918 || opline->opcode == ZEND_SUB
2919 || opline->opcode == ZEND_MUL
2920 || opline->opcode == ZEND_FETCH_DIM_R
2921 || opline->opcode == ZEND_FETCH_CONSTANT) {
2922 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
2923 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
2924 start[ssa_op->result_def] = idx;
2925 vars_op_array[ssa_op->result_def] = op_array;
2926 count++;
2927 }
2928 }
2929 }
2930 if (ssa_op->op1_def >= 0
2931 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2932 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2933 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2934 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2935 start[ssa_op->op1_def] = idx;
2936 vars_op_array[ssa_op->op1_def] = op_array;
2937 count++;
2938 }
2939 if (ssa_op->op2_def >= 0
2940 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
2941 || ssa->vars[ssa_op->op2_def].phi_use_chain)
2942 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
2943 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)) {
2944 start[ssa_op->op2_def] = idx;
2945 vars_op_array[ssa_op->op2_def] = op_array;
2946 count++;
2947 }
2948 }
2949
2950 len = zend_jit_trace_op_len(opline);
2951 switch (opline->opcode) {
2952 case ZEND_ASSIGN_DIM:
2953 case ZEND_ASSIGN_OBJ:
2954 case ZEND_ASSIGN_STATIC_PROP:
2955 case ZEND_ASSIGN_DIM_OP:
2956 case ZEND_ASSIGN_OBJ_OP:
2957 case ZEND_ASSIGN_STATIC_PROP_OP:
2958 case ZEND_ASSIGN_OBJ_REF:
2959 case ZEND_ASSIGN_STATIC_PROP_REF:
2960 /* OP_DATA */
2961 ssa_op++;
2962 opline++;
2963 if (ssa_op->op1_use >= 0
2964 && start[ssa_op->op1_use] >= 0
2965 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2966 if (support_opline) {
2967 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2968 if (opline->op1_type != IS_CV) {
2969 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2970 }
2971 } else {
2972 start[ssa_op->op1_use] = -1;
2973 end[ssa_op->op1_use] = -1;
2974 count--;
2975 }
2976 }
2977 if (ssa_op->op1_def >= 0) {
2978 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2979 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2980 if (support_opline
2981 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2982 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2983 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2984 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2985 start[ssa_op->op1_def] = idx;
2986 vars_op_array[ssa_op->op1_def] = op_array;
2987 count++;
2988 }
2989 }
2990 ssa_op++;
2991 opline++;
2992 idx+=2;
2993 break;
2994 case ZEND_RECV_INIT:
2995 ssa_op++;
2996 opline++;
2997 idx++;
2998 while (opline->opcode == ZEND_RECV_INIT) {
2999 /* RECV_INIT doesn't support registers */
3000 if (ssa_op->result_def >= 0) {
3001 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
3002 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3003 }
3004 ssa_op++;
3005 opline++;
3006 idx++;
3007 }
3008 break;
3009 case ZEND_BIND_GLOBAL:
3010 ssa_op++;
3011 opline++;
3012 idx++;
3013 while (opline->opcode == ZEND_BIND_GLOBAL) {
3014 /* BIND_GLOBAL doesn't support registers */
3015 if (ssa_op->op1_def >= 0) {
3016 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
3017 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3018 }
3019 ssa_op++;
3020 opline++;
3021 idx++;
3022 }
3023 break;
3024 default:
3025 ssa_op += len;
3026 idx += len;
3027 break;
3028 }
3029 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3030 /* New call frames */
3031 zend_jit_trace_stack_frame *prev_frame = frame;
3032
3033 frame = zend_jit_trace_call_frame(frame, op_array);
3034 frame->prev = prev_frame;
3035 frame->func = (const zend_function*)p->op_array;
3036 stack = frame->stack;
3037 op_array = p->op_array;
3038 jit_extension =
3039 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3040 op_array_ssa = &jit_extension->func_info.ssa;
3041 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3042 for (i = 0; i < op_array->last_var; i++) {
3043 SET_STACK_VAR(stack, i, j);
3044 vars_op_array[j] = op_array;
3045 if (ssa->vars[j].use_chain >= 0
3046 && ssa->vars[j].alias == NO_ALIAS
3047 && zend_jit_var_supports_reg(ssa, j)) {
3048 start[j] = idx;
3049 flags[j] = ZREG_LOAD;
3050 count++;
3051 }
3052 j++;
3053 }
3054 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3055 SET_STACK_VAR(stack, i, -1);
3056 }
3057 level++;
3058 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3059 /* Close exiting call frames */
3060 for (i = 0; i < op_array->last_var; i++) {
3061 zend_jit_close_var(stack, i, start, end, flags, idx-1);
3062 }
3063 op_array = p->op_array;
3064 jit_extension =
3065 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3066 op_array_ssa = &jit_extension->func_info.ssa;
3067 frame = zend_jit_trace_ret_frame(frame, op_array);
3068 stack = frame->stack;
3069 if (level == 0) {
3070 /* New return frames */
3071 frame->prev = NULL;
3072 frame->func = (const zend_function*)op_array;
3073 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3074 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3075 SET_STACK_VAR(stack, i, j);
3076 vars_op_array[j] = op_array;
3077 if (ssa->vars[j].use_chain >= 0
3078 && ssa->vars[j].alias == NO_ALIAS
3079 && zend_jit_var_supports_reg(ssa, j)) {
3080 start[j] = idx;
3081 flags[j] = ZREG_LOAD;
3082 count++;
3083 }
3084 j++;
3085 }
3086 } else {
3087 level--;
3088 }
3089 } else if (p->op == ZEND_JIT_TRACE_END) {
3090 break;
3091 }
3092 }
3093
3094 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3095 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3096 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3097 zend_ssa_phi *phi = ssa->blocks[1].phis;
3098
3099 while (phi) {
3100 i = phi->sources[1];
3101 if (start[i] >= 0 && !ssa->vars[phi->ssa_var].no_val) {
3102 end[i] = idx;
3103 flags[i] &= ~ZREG_LAST_USE;
3104 }
3105 phi = phi->next;
3106 }
3107
3108 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3109 for (i = 0; i < op_array->last_var; i++) {
3110 if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) {
3111 end[i] = idx;
3112 flags[i] &= ~ZREG_LAST_USE;
3113 } else {
3114 zend_jit_close_var(stack, i, start, end, flags, idx);
3115 }
3116 }
3117 }
3118 } else {
3119 last_idx = idx;
3120 for (i = 0; i < op_array->last_var; i++) {
3121 zend_jit_close_var(stack, i, start, end, flags, idx);
3122 }
3123 while (frame->prev) {
3124 frame = frame->prev;
3125 op_array = &frame->func->op_array;
3126 stack = frame->stack;
3127 for (i = 0; i < op_array->last_var; i++) {
3128 zend_jit_close_var(stack, i, start, end, flags, idx);
3129 }
3130 }
3131 }
3132
3133 if (!count) {
3134 free_alloca(start, use_heap);
3135 zend_arena_release(&CG(arena), checkpoint);
3136 return NULL;
3137 }
3138
3139 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval));
3140 memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count);
3141 list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count);
3142 j = 0;
3143 for (i = 0; i < ssa->vars_count; i++) {
3144 if (start[i] >= 0 && end[i] >= 0) {
3145 ZEND_ASSERT(j < count);
3146 if ((flags[i] & ZREG_LOAD) &&
3147 (flags[i] & ZREG_LAST_USE) &&
3148 end[i] == ssa->vars[i].use_chain) {
3149 /* skip life range with single use */
3150 continue;
3151 }
3152 intervals[i] = &list[j];
3153 list[j].ssa_var = i;
3154 list[j].reg = ZREG_NONE;
3155 list[j].flags = flags[i];
3156 list[j].range.start = start[i];
3157 list[j].range.end = end[i];
3158 list[j].range.next = NULL;
3159 list[j].hint = NULL;
3160 list[j].used_as_hint = NULL;
3161 list[j].list_next = NULL;
3162 j++;
3163 }
3164 }
3165 count = j;
3166 free_alloca(start, use_heap);
3167 start = end = NULL;
3168
3169 if (!count) {
3170 zend_arena_release(&CG(arena), checkpoint);
3171 return NULL;
3172 }
3173
3174 /* Add hints */
3175 if (parent_vars_count) {
3176 i = trace_buffer->op_array->last_var;
3177 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3178 i += trace_buffer->op_array->T;
3179 }
3180 if ((uint32_t)i > parent_vars_count) {
3181 i = parent_vars_count;
3182 }
3183 while (i > 0) {
3184 i--;
3185 if (intervals[i]
3186 && STACK_REG(parent_stack, i) != ZREG_NONE
3187 && STACK_REG(parent_stack, i) < ZREG_NUM) {
3188 list[j].ssa_var = - 1;
3189 list[j].reg = STACK_REG(parent_stack, i);
3190 list[j].flags = 0;
3191 list[j].range.start = -1;
3192 list[j].range.end = -1;
3193 list[j].range.next = NULL;
3194 list[j].hint = NULL;
3195 list[j].used_as_hint = NULL;
3196 list[j].list_next = NULL;
3197 intervals[i]->hint = &list[j];
3198 j++;
3199 }
3200 }
3201 }
3202
3203 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3204 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3205 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3206 zend_ssa_phi *phi = ssa->blocks[1].phis;
3207
3208 while (phi) {
3209 if (intervals[phi->ssa_var]) {
3210 if (intervals[phi->sources[1]]
3211 && (ssa->var_info[phi->sources[1]].type & MAY_BE_ANY) ==
3212 (ssa->var_info[phi->ssa_var].type & MAY_BE_ANY)) {
3213 intervals[phi->sources[1]]->hint = intervals[phi->ssa_var];
3214 }
3215 }
3216 phi = phi->next;
3217 }
3218 }
3219
3220 for (i = 0; i < ssa->vars_count; i++) {
3221 if (intervals[i] && !intervals[i]->hint) {
3222
3223 if (ssa->vars[i].definition >= 0) {
3224 uint32_t line = ssa->vars[i].definition;
3225 const zend_op *opline = ssa_opcodes[line];
3226
3227 switch (opline->opcode) {
3228 case ZEND_QM_ASSIGN:
3229 case ZEND_POST_INC:
3230 case ZEND_POST_DEC:
3231 if (ssa->ops[line].op1_use >= 0 &&
3232 intervals[ssa->ops[line].op1_use] &&
3233 (i == ssa->ops[line].op1_def ||
3234 (i == ssa->ops[line].result_def &&
3235 (ssa->ops[line].op1_def < 0 ||
3236 !intervals[ssa->ops[line].op1_def])))) {
3237 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3238 }
3239 break;
3240 case ZEND_SEND_VAR:
3241 if (opline->op2_type == IS_CONST) {
3242 /* Named parameters not supported in JIT */
3243 break;
3244 }
3245 case ZEND_PRE_INC:
3246 case ZEND_PRE_DEC:
3247 if (i == ssa->ops[line].op1_def &&
3248 ssa->ops[line].op1_use >= 0 &&
3249 intervals[ssa->ops[line].op1_use]) {
3250 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3251 }
3252 break;
3253 case ZEND_ASSIGN:
3254 if (ssa->ops[line].op2_use >= 0 &&
3255 intervals[ssa->ops[line].op2_use] &&
3256 (i == ssa->ops[line].op2_def ||
3257 (i == ssa->ops[line].op1_def &&
3258 (ssa->ops[line].op2_def < 0 ||
3259 !intervals[ssa->ops[line].op2_def])) ||
3260 (i == ssa->ops[line].result_def &&
3261 (ssa->ops[line].op2_def < 0 ||
3262 !intervals[ssa->ops[line].op2_def]) &&
3263 (ssa->ops[line].op1_def < 0 ||
3264 !intervals[ssa->ops[line].op1_def])))) {
3265 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3266 }
3267 break;
3268 case ZEND_SUB:
3269 case ZEND_ADD:
3270 case ZEND_MUL:
3271 case ZEND_BW_OR:
3272 case ZEND_BW_AND:
3273 case ZEND_BW_XOR:
3274 if (i == ssa->ops[line].result_def) {
3275 if (ssa->ops[line].op1_use >= 0 &&
3276 intervals[ssa->ops[line].op1_use] &&
3277 ssa->ops[line].op1_use_chain < 0 &&
3278 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
3279 (ssa->var_info[i].type & MAY_BE_ANY) ==
3280 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
3281
3282 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op1_use].definition_phi;
3283 if (phi &&
3284 intervals[phi->sources[1]] &&
3285 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op1_use]) {
3286 break;
3287 }
3288 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3289 } else if (opline->opcode != ZEND_SUB &&
3290 ssa->ops[line].op2_use >= 0 &&
3291 intervals[ssa->ops[line].op2_use] &&
3292 ssa->ops[line].op2_use_chain < 0 &&
3293 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
3294 (ssa->var_info[i].type & MAY_BE_ANY) ==
3295 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
3296
3297 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op2_use].definition_phi;
3298 if (phi &&
3299 intervals[phi->sources[1]] &&
3300 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op2_use]) {
3301 break;
3302 }
3303 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3304 }
3305 }
3306 break;
3307 }
3308 }
3309 }
3310 }
3311
3312 list = zend_jit_sort_intervals(intervals, ssa->vars_count);
3313
3314 if (list) {
3315 ival = list;
3316 while (ival) {
3317 if (ival->hint) {
3318 ival->hint->used_as_hint = ival;
3319 }
3320 ival = ival->list_next;
3321 }
3322 }
3323
3324 if (list) {
3325 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3326 fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM);
3327 ival = list;
3328 while (ival) {
3329 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3330 ival = ival->list_next;
3331 }
3332 }
3333 }
3334
3335 /* Linear Scan Register Allocation (op_array is not actually used, only fn_flags matters) */
3336 list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list);
3337
3338 if (list) {
3339 zend_lifetime_interval *ival, *next;
3340
3341 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
3342 ival = list;
3343 count = 0;
3344 while (ival != NULL) {
3345 ZEND_ASSERT(ival->reg != ZREG_NONE);
3346 count++;
3347 next = ival->list_next;
3348 ival->list_next = intervals[ival->ssa_var];
3349 intervals[ival->ssa_var] = ival;
3350 ival = next;
3351 }
3352
3353 if (!count) {
3354 zend_arena_release(&CG(arena), checkpoint);
3355 return NULL;
3356 }
3357
3358 /* Add LOAD flag to registers that can't reuse register from parent trace */
3359 if (parent_vars_count) {
3360 i = trace_buffer->op_array->last_var;
3361 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3362 i += trace_buffer->op_array->T;
3363 }
3364 if ((uint32_t)i > parent_vars_count) {
3365 i = parent_vars_count;
3366 }
3367 while (i > 0) {
3368 i--;
3369 if (intervals[i] && intervals[i]->reg != STACK_REG(parent_stack, i)) {
3370 intervals[i]->flags |= ZREG_LOAD;
3371 }
3372 }
3373 }
3374
3375 /* SSA resolution */
3376 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3377 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3378 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3379 zend_ssa_phi *phi = ssa->blocks[1].phis;
3380
3381 while (phi) {
3382 int def = phi->ssa_var;
3383 int use = phi->sources[1];
3384
3385 if (intervals[def]) {
3386 if (!intervals[use]) {
3387 intervals[def]->flags |= ZREG_LOAD;
3388 if ((intervals[def]->flags & ZREG_LAST_USE)
3389 && ssa->vars[def].use_chain >= 0
3390 && ssa->vars[def].use_chain == intervals[def]->range.end) {
3391 /* remove interval used once */
3392 intervals[def] = NULL;
3393 count--;
3394 }
3395 } else if (intervals[def]->reg != intervals[use]->reg) {
3396 intervals[def]->flags |= ZREG_LOAD;
3397 if (ssa->vars[use].use_chain >= 0) {
3398 intervals[use]->flags |= ZREG_STORE;
3399 } else {
3400 intervals[use] = NULL;
3401 count--;
3402 }
3403 } else {
3404 use = phi->sources[0];
3405 ZEND_ASSERT(!intervals[use]);
3406 intervals[use] = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
3407 intervals[use]->ssa_var = phi->sources[0];
3408 intervals[use]->reg = intervals[def]->reg;
3409 intervals[use]->flags = ZREG_LOAD;
3410 intervals[use]->range.start = 0;
3411 intervals[use]->range.end = 0;
3412 intervals[use]->range.next = NULL;
3413 intervals[use]->hint = NULL;
3414 intervals[use]->used_as_hint = NULL;
3415 intervals[use]->list_next = NULL;
3416 }
3417 } else if (intervals[use]
3418 && (!ssa->vars[def].no_val
3419 || ssa->var_info[def].type != ssa->var_info[use].type)) {
3420 if (ssa->vars[use].use_chain >= 0) {
3421 intervals[use]->flags |= ZREG_STORE;
3422 } else {
3423 intervals[use] = NULL;
3424 count--;
3425 }
3426 }
3427 phi = phi->next;
3428 }
3429 } else {
3430 for (i = 0; i < ssa->vars_count; i++) {
3431 if (intervals[i]
3432 && intervals[i]->range.end == last_idx
3433 && !(intervals[i]->flags & (ZREG_LOAD|ZREG_STORE))) {
3434 intervals[i]->flags |= ZREG_STORE;
3435 }
3436 }
3437 }
3438
3439 if (!count) {
3440 zend_arena_release(&CG(arena), checkpoint);
3441 return NULL;
3442 }
3443
3444 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3445 fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM);
3446 for (i = 0; i < ssa->vars_count; i++) {
3447 ival = intervals[i];
3448 while (ival) {
3449 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3450 ival = ival->list_next;
3451 }
3452 }
3453 }
3454
3455 return intervals;
3456 }
3457
3458 zend_arena_release(&CG(arena), checkpoint); //???
3459 return NULL;
3460 }
3461
zend_jit_trace_clenup_stack(zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,zend_lifetime_interval ** ra)3462 static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, zend_lifetime_interval **ra)
3463 {
3464 uint32_t line = ssa_op - ssa->ops;
3465
3466 if (ssa_op->op1_use >= 0
3467 && ra[ssa_op->op1_use]
3468 && ra[ssa_op->op1_use]->range.end == line) {
3469 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
3470 }
3471 if (ssa_op->op2_use >= 0
3472 && ra[ssa_op->op2_use]
3473 && ra[ssa_op->op2_use]->range.end == line) {
3474 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op2.var), ZREG_NONE);
3475 }
3476 if (ssa_op->result_use >= 0
3477 && ra[ssa_op->result_use]
3478 && ra[ssa_op->result_use]->range.end == line) {
3479 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE);
3480 }
3481 }
3482
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3483 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3484 {
3485 zend_op *next_opline = (zend_op*)(opline + 1);
3486
3487 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3488 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3489 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3490 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3491 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3492 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3493 }
3494 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3495 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3496 }
3497 }
3498
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3499 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3500 {
3501 int var = ssa_op->result_def;
3502 int i;
3503 int use = ssa->vars[var].use_chain;
3504 const zend_op *opline;
3505
3506 if (use < 0
3507 || ssa->vars[var].phi_use_chain
3508 || ssa->ops[use].op1_use != var
3509 || ssa->ops[use].op1_use_chain != -1) {
3510 return 0;
3511 }
3512
3513 opline = ssa_opcodes[use];
3514 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3515 return (opline->op2_type == IS_CONST &&
3516 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3517 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3518 if (!JIT_G(current_frame)
3519 || !JIT_G(current_frame)->call
3520 || !JIT_G(current_frame)->call->func
3521 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3522 return 0;
3523 }
3524 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3525 && opline->opcode != ZEND_FETCH_OBJ_IS
3526 && opline->opcode != ZEND_FETCH_OBJ_W
3527 && opline->opcode != ZEND_ASSIGN_OBJ
3528 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3529 && opline->opcode != ZEND_PRE_INC_OBJ
3530 && opline->opcode != ZEND_PRE_DEC_OBJ
3531 && opline->opcode != ZEND_POST_INC_OBJ
3532 && opline->opcode != ZEND_POST_DEC_OBJ) {
3533 return 0;
3534 }
3535
3536 if (opline->op2_type != IS_CONST
3537 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3538 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3539 return 0;
3540 }
3541
3542 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3543 if (opline->op1_type == IS_CV
3544 && (opline+1)->op1_type == IS_CV
3545 && (opline+1)->op1.var == opline->op1.var) {
3546 /* skip $a->prop += $a; */
3547 return 0;
3548 }
3549 if (!zend_jit_supported_binary_op(
3550 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3551 return 0;
3552 }
3553 }
3554
3555 for (i = ssa->vars[var].definition; i < use; i++) {
3556 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3557 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3558 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3559 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3560 return 0;
3561 }
3562 }
3563
3564 return 1;
3565 }
3566
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3567 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3568 {
3569 uint32_t i;
3570
3571 for (i = 0; i < stack_size; i++) {
3572 if (STACK_REG(stack, i) != ZREG_NONE
3573 && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
3574 return 1;
3575 }
3576 }
3577 return 0;
3578 }
3579
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3580 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3581 {
3582 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3583 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3584 uint32_t stack_size;
3585 zend_jit_trace_stack *stack;
3586
3587 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3588 return 1;
3589 }
3590
3591 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3592 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3593 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3594 }
3595
zend_jit_trace_deoptimization(dasm_State ** Dst,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_lifetime_interval ** ra,bool polymorphic_side_trace)3596 static int zend_jit_trace_deoptimization(dasm_State **Dst,
3597 uint32_t flags,
3598 const zend_op *opline,
3599 zend_jit_trace_stack *parent_stack,
3600 int parent_vars_count,
3601 zend_ssa *ssa,
3602 zend_jit_trace_stack *stack,
3603 zend_lifetime_interval **ra,
3604 bool polymorphic_side_trace)
3605 {
3606 int i;
3607 bool has_constants = 0;
3608 bool has_unsaved_vars = 0;
3609
3610 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3611 for (i = 0; i < parent_vars_count; i++) {
3612 int8_t reg = STACK_REG(parent_stack, i);
3613
3614 if (reg != ZREG_NONE) {
3615 if (reg < ZREG_NUM) {
3616 if (ssa && ssa->vars[i].no_val) {
3617 /* pass */
3618 } else if (ra && ra[i] && ra[i]->reg == reg) {
3619 /* register already loaded by parent trace */
3620 if (stack) {
3621 SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i));
3622 }
3623 has_unsaved_vars = 1;
3624 } else {
3625 uint8_t type = STACK_TYPE(parent_stack, i);
3626
3627 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3628 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3629 STACK_MEM_TYPE(parent_stack, i) != type)) {
3630 return 0;
3631 }
3632 if (stack) {
3633 SET_STACK_TYPE(stack, i, type, 1);
3634 }
3635 }
3636 } else {
3637 /* delay custom deoptimization instructions to prevent register clobbering */
3638 has_constants = 1;
3639 }
3640 }
3641 }
3642
3643 if (has_unsaved_vars
3644 && (has_constants
3645 || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) {
3646 for (i = 0; i < parent_vars_count; i++) {
3647 int8_t reg = STACK_REG(parent_stack, i);
3648
3649 if (reg != ZREG_NONE) {
3650 if (reg < ZREG_NUM) {
3651 if (ssa && ssa->vars[i].no_val) {
3652 /* pass */
3653 } else if (ra && ra[i] && ra[i]->reg == reg) {
3654 uint8_t type = STACK_TYPE(parent_stack, i);
3655
3656 if (stack) {
3657 SET_STACK_TYPE(stack, i, type, 1);
3658 }
3659 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3660 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3661 STACK_MEM_TYPE(parent_stack, i) != type)) {
3662 return 0;
3663 }
3664 }
3665 }
3666 }
3667 }
3668 }
3669
3670 if (has_constants) {
3671 for (i = 0; i < parent_vars_count; i++) {
3672 int8_t reg = STACK_REG(parent_stack, i);
3673
3674 if (reg != ZREG_NONE) {
3675 if (reg < ZREG_NUM) {
3676 /* pass */
3677 } else if (reg == ZREG_THIS) {
3678 if (polymorphic_side_trace) {
3679 ssa->var_info[i].delayed_fetch_this = 1;
3680 if (stack) {
3681 SET_STACK_REG(stack, i, ZREG_THIS);
3682 }
3683 } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
3684 return 0;
3685 }
3686 } else {
3687 if (reg == ZREG_ZVAL_COPY_GPR0
3688 &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) {
3689 return 0;
3690 }
3691 if (!zend_jit_store_const(Dst, i, reg)) {
3692 return 0;
3693 }
3694 }
3695 }
3696 }
3697 }
3698
3699 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3700 if (!zend_jit_save_call_chain(Dst, -1)) {
3701 return 0;
3702 }
3703 }
3704
3705 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3706 const zend_op *op = opline - 1;
3707
3708 if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) {
3709 return 0;
3710 }
3711 }
3712
3713 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3714 const zend_op *op = opline - 1;
3715
3716 if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) {
3717 return 0;
3718 }
3719 }
3720
3721 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3722 if (!zend_jit_check_exception(Dst)) {
3723 return 0;
3724 }
3725 }
3726
3727 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3728 if (!zend_jit_free_trampoline(Dst)) {
3729 return 0;
3730 }
3731 }
3732
3733 return 1;
3734 }
3735
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3736 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3737 {
3738 info->has_range = 1;
3739 info->range.min = min;
3740 info->range.max = max;
3741 info->range.underflow = 0;
3742 info->range.overflow = 0;
3743 }
3744
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3745 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3746 {
3747 zend_long op1_min, op1_max, op2_min, op2_max;
3748
3749 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3750 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3751 return;
3752 }
3753
3754 op1_min = OP1_MIN_RANGE();
3755 op1_max = OP1_MAX_RANGE();
3756 op2_min = OP2_MIN_RANGE();
3757 op2_max = OP2_MAX_RANGE();
3758
3759 switch (opline->opcode) {
3760 case ZEND_IS_EQUAL:
3761 case ZEND_CASE:
3762 case ZEND_IS_IDENTICAL:
3763 case ZEND_CASE_STRICT:
3764 case ZEND_IS_NOT_IDENTICAL:
3765 if (!exit_if_true) {
3766 /* op1 == op2 */
3767 if (ssa_op->op1_use >= 0) {
3768 zend_jit_trace_set_var_range(
3769 &ssa->var_info[ssa_op->op1_use],
3770 MAX(op1_min, op2_min),
3771 MIN(op1_max, op2_max));
3772 }
3773 if (ssa_op->op2_use >= 0) {
3774 zend_jit_trace_set_var_range(
3775 &ssa->var_info[ssa_op->op2_use],
3776 MAX(op2_min, op1_min),
3777 MIN(op2_max, op1_max));
3778 }
3779 }
3780 break;
3781 case ZEND_IS_NOT_EQUAL:
3782 if (exit_if_true) {
3783 /* op1 == op2 */
3784 if (ssa_op->op1_use >= 0) {
3785 zend_jit_trace_set_var_range(
3786 &ssa->var_info[ssa_op->op1_use],
3787 MAX(op1_min, op2_min),
3788 MIN(op1_max, op2_max));
3789 }
3790 if (ssa_op->op2_use >= 0) {
3791 zend_jit_trace_set_var_range(
3792 &ssa->var_info[ssa_op->op2_use],
3793 MAX(op2_min, op1_min),
3794 MIN(op2_max, op1_max));
3795 }
3796 }
3797 break;
3798 case ZEND_IS_SMALLER_OR_EQUAL:
3799 if (!exit_if_true) {
3800 /* op1 <= op2 */
3801 if (ssa_op->op1_use >= 0) {
3802 zend_jit_trace_set_var_range(
3803 &ssa->var_info[ssa_op->op1_use],
3804 op1_min,
3805 MIN(op1_max, op2_max));
3806 }
3807 if (ssa_op->op2_use >= 0) {
3808 zend_jit_trace_set_var_range(
3809 &ssa->var_info[ssa_op->op2_use],
3810 MAX(op2_min, op1_min),
3811 op2_max);
3812 }
3813 } else {
3814 /* op1 > op2 */
3815 if (ssa_op->op1_use >= 0) {
3816 zend_jit_trace_set_var_range(
3817 &ssa->var_info[ssa_op->op1_use],
3818 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3819 op1_max);
3820 }
3821 if (ssa_op->op2_use >= 0) {
3822 zend_jit_trace_set_var_range(
3823 &ssa->var_info[ssa_op->op2_use],
3824 op2_min,
3825 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3826 }
3827 }
3828 break;
3829 case ZEND_IS_SMALLER:
3830 if (!exit_if_true) {
3831 /* op1 < op2 */
3832 if (ssa_op->op1_use >= 0) {
3833 zend_jit_trace_set_var_range(
3834 &ssa->var_info[ssa_op->op1_use],
3835 op1_min,
3836 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3837 }
3838 if (ssa_op->op2_use >= 0) {
3839 zend_jit_trace_set_var_range(
3840 &ssa->var_info[ssa_op->op2_use],
3841 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3842 op2_max);
3843 }
3844 } else {
3845 /* op1 >= op2 */
3846 if (ssa_op->op1_use >= 0) {
3847 zend_jit_trace_set_var_range(
3848 &ssa->var_info[ssa_op->op1_use],
3849 MAX(op1_min, op2_min),
3850 op1_max);
3851 }
3852 if (ssa_op->op2_use >= 0) {
3853 zend_jit_trace_set_var_range(
3854 &ssa->var_info[ssa_op->op2_use],
3855 op2_min,
3856 MIN(op2_max, op1_max));
3857 }
3858 }
3859 break;
3860 }
3861 }
3862
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3863 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3864 {
3865 uint8_t prev_opcode;
3866
3867 if (opline->op1_type == IS_CONST
3868 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3869 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3870 if (ssa_op->op2_use >= 0) {
3871 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3872 ssa_op--;
3873 opline = ssa_opcodes[ssa_op - ssa->ops];
3874 prev_opcode = opline->opcode;
3875 if (prev_opcode == ZEND_PRE_INC
3876 || prev_opcode == ZEND_PRE_DEC
3877 || prev_opcode == ZEND_POST_INC
3878 || prev_opcode == ZEND_POST_DEC) {
3879 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3880 }
3881 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3882 ssa_op--;
3883 opline = ssa_opcodes[ssa_op - ssa->ops];
3884 prev_opcode = opline->opcode;
3885 if (prev_opcode == ZEND_ADD
3886 || prev_opcode == ZEND_SUB) {
3887 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3888 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3889 }
3890 }
3891 }
3892 } else if (opline->op2_type == IS_CONST
3893 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3894 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3895 if (ssa_op->op1_use >= 0) {
3896 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3897 ssa_op--;
3898 opline = ssa_opcodes[ssa_op - ssa->ops];
3899 prev_opcode = opline->opcode;
3900 if (prev_opcode == ZEND_PRE_INC
3901 || prev_opcode == ZEND_PRE_DEC
3902 || prev_opcode == ZEND_POST_INC
3903 || prev_opcode == ZEND_POST_DEC) {
3904 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3905 }
3906 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3907 ssa_op--;
3908 opline = ssa_opcodes[ssa_op - ssa->ops];
3909 prev_opcode = opline->opcode;
3910 if (prev_opcode == ZEND_ADD
3911 || prev_opcode == ZEND_SUB) {
3912 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3913 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3914 }
3915 }
3916 }
3917 } else {
3918 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3919 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3920
3921 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3922 && prev_ssa_op != ssa->ops
3923 && prev_ssa_op->op1_use >= 0
3924 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3925 prev_ssa_op--;
3926 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3927 }
3928
3929 if (ssa_op->op1_use == prev_ssa_op->op1_use
3930 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3931 if (prev_opcode == ZEND_IS_EQUAL
3932 || prev_opcode == ZEND_IS_NOT_EQUAL
3933 || prev_opcode == ZEND_IS_SMALLER
3934 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3935 || prev_opcode == ZEND_CASE
3936 || prev_opcode == ZEND_IS_IDENTICAL
3937 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3938 || prev_opcode == ZEND_CASE_STRICT) {
3939 if (ssa_op->op1_use < 0) {
3940 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3941 return 0;
3942 }
3943 }
3944 if (ssa_op->op2_use < 0) {
3945 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3946 return 0;
3947 }
3948 }
3949 return 1;
3950 }
3951 }
3952 }
3953 return 0;
3954 }
3955
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3956 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3957 zend_jit_trace_rec *p,
3958 zend_jit_trace_stack_frame *frame)
3959 {
3960 if (opline->result_type == IS_TMP_VAR
3961 && (p+1)->op == ZEND_JIT_TRACE_VM
3962 && (p+1)->opline == opline + 1
3963 && ((opline+1)->opcode == ZEND_SEND_VAL
3964 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3965 && frame
3966 && frame->call
3967 && frame->call->func
3968 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3969 && (opline+1)->op1_type == IS_TMP_VAR
3970 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3971 && (opline+1)->op1.var == opline->result.var) {
3972
3973 if (frame->call && frame->call->func) {
3974 uint8_t res_type = (p+1)->op1_type;
3975
3976 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3977 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3978 }
3979 }
3980 return 1;
3981 }
3982 return 0;
3983 }
3984
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)3985 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
3986 {
3987 const void *handler = NULL;
3988 dasm_State* dasm_state = NULL;
3989 zend_script *script = NULL;
3990 zend_lifetime_interval **ra = NULL;
3991 zend_string *name = NULL;
3992 void *checkpoint;
3993 const zend_op_array *op_array;
3994 zend_ssa *ssa, *op_array_ssa;
3995 const zend_op **ssa_opcodes;
3996 zend_jit_trace_rec *p;
3997 zend_jit_op_array_trace_extension *jit_extension;
3998 int num_op_arrays = 0;
3999 zend_jit_trace_info *t;
4000 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
4001 uint8_t smart_branch_opcode;
4002 const void *exit_addr;
4003 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info;
4004 bool send_result = 0;
4005 bool skip_comparison;
4006 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
4007 zend_class_entry *ce;
4008 bool ce_is_instanceof;
4009 bool on_this = 0;
4010 bool delayed_fetch_this = 0;
4011 bool avoid_refcounting = 0;
4012 bool polymorphic_side_trace =
4013 parent_trace &&
4014 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
4015 uint32_t i;
4016 zend_jit_trace_stack_frame *frame, *top, *call;
4017 zend_jit_trace_stack *stack;
4018 uint8_t res_type = IS_UNKNOWN;
4019 const zend_op *opline, *orig_opline;
4020 const zend_ssa_op *ssa_op, *orig_ssa_op;
4021 int checked_stack;
4022 int peek_checked_stack;
4023 uint32_t frame_flags = 0;
4024
4025 JIT_G(current_trace) = trace_buffer;
4026
4027 checkpoint = zend_arena_checkpoint(CG(arena));
4028
4029 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4030
4031 if (!ssa) {
4032 goto jit_cleanup;
4033 }
4034
4035 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4036
4037 /* Register allocation */
4038 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4039 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4040 ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4041 }
4042
4043 p = trace_buffer;
4044 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4045 op_array = p->op_array;
4046 frame = JIT_G(current_frame);
4047 top = zend_jit_trace_call_frame(frame, op_array);
4048 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4049 frame->used_stack = checked_stack = peek_checked_stack = 0;
4050 stack = frame->stack;
4051 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4052 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4053 }
4054
4055 opline = p[1].opline;
4056 name = zend_jit_trace_name(op_array, opline->lineno);
4057 p += ZEND_JIT_TRACE_START_REC_SIZE;
4058
4059 dasm_init(&dasm_state, DASM_MAXSECTION);
4060 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4061 dasm_setup(&dasm_state, dasm_actions);
4062
4063 jit_extension =
4064 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4065 op_array_ssa = &jit_extension->func_info.ssa;
4066
4067 dasm_growpc(&dasm_state, 2); /* =>0: loop header */
4068 /* =>1: end of code */
4069
4070 zend_jit_align_func(&dasm_state);
4071 if (!parent_trace) {
4072 zend_jit_prologue(&dasm_state);
4073 }
4074 zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM,
4075 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4076
4077 if (!parent_trace) {
4078 zend_jit_set_last_valid_opline(opline);
4079 zend_jit_track_last_valid_opline();
4080 } else {
4081 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4082 zend_jit_trace_opline_guard(&dasm_state, opline);
4083 } else {
4084 zend_jit_reset_last_valid_opline();
4085 }
4086 }
4087
4088 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4089 int last_var;
4090 int parent_vars_count = 0;
4091 zend_jit_trace_stack *parent_stack = NULL;
4092 int used_stack = ((zend_tssa*)ssa)->used_stack;
4093
4094 if (used_stack > 0) {
4095 peek_checked_stack = used_stack;
4096 if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) {
4097 goto jit_failure;
4098 }
4099 }
4100
4101 if (parent_trace) {
4102 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4103 op_array->last_var + op_array->T);
4104 if (parent_vars_count) {
4105 parent_stack =
4106 zend_jit_traces[parent_trace].stack_map +
4107 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4108 }
4109 }
4110
4111 last_var = op_array->last_var;
4112 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4113 last_var += op_array->T;
4114 }
4115
4116 for (i = 0; i < last_var; i++) {
4117 uint32_t info = ssa->var_info[i].type;
4118
4119 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4120 uint8_t type, mem_type;
4121
4122 type = concrete_type(info);
4123 if (i < parent_vars_count
4124 && STACK_TYPE(parent_stack, i) == type) {
4125 mem_type = STACK_MEM_TYPE(parent_stack, i);
4126 if (mem_type != IS_UNKNOWN) {
4127 SET_STACK_TYPE(stack, i, mem_type, 1);
4128 }
4129 SET_STACK_TYPE(stack, i, type, 0);
4130 } else {
4131 SET_STACK_TYPE(stack, i, type, 1);
4132 }
4133 } else if (ssa->vars[i].alias != NO_ALIAS) {
4134 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4135 } else if (i < parent_vars_count
4136 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4137 /* This must be already handled by trace type inference */
4138 ZEND_UNREACHABLE();
4139 // SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
4140 } else if ((info & MAY_BE_GUARD) != 0
4141 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4142 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4143 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4144 && (opline-1)->result_type == IS_VAR
4145 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4146 && (ssa->vars[i].use_chain != -1
4147 || (ssa->vars[i].phi_use_chain
4148 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4149 /* Check loop-invariant variable type */
4150 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4151 goto jit_failure;
4152 }
4153 info &= ~MAY_BE_GUARD;
4154 ssa->var_info[i].type = info;
4155 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4156 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4157 && op_array->function_name
4158 && i >= op_array->num_args) {
4159 /* This must be already handled by trace type inference */
4160 ZEND_UNREACHABLE();
4161 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4162 }
4163
4164 if ((info & MAY_BE_PACKED_GUARD) != 0
4165 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4166 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4167 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4168 && (ssa->vars[i].use_chain != -1
4169 || (ssa->vars[i].phi_use_chain
4170 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4171 if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) {
4172 goto jit_failure;
4173 }
4174 info &= ~MAY_BE_PACKED_GUARD;
4175 ssa->var_info[i].type = info;
4176 }
4177 }
4178
4179 if (parent_trace) {
4180 /* Deoptimization */
4181 if (!zend_jit_trace_deoptimization(&dasm_state,
4182 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4183 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4184 parent_stack, parent_vars_count, ssa, stack, ra,
4185 polymorphic_side_trace)) {
4186 goto jit_failure;
4187 }
4188 }
4189
4190 if (ra
4191 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4192 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4193 for (i = 0; i < last_var; i++) {
4194 if (ra[i]
4195 && (ra[i]->flags & ZREG_LOAD) != 0
4196 && ra[i]->reg != stack[i].reg) {
4197
4198 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4199 uint8_t op_type;
4200
4201 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4202 op_type = concrete_type(ssa->var_info[i].type);
4203 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
4204 goto jit_failure;
4205 }
4206 SET_STACK_TYPE(stack, i, op_type, 1);
4207 }
4208
4209 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4210 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4211 goto jit_failure;
4212 }
4213 }
4214 }
4215 }
4216 }
4217
4218 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4219 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4220 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4221
4222 zend_jit_label(&dasm_state, 0); /* start of of trace loop */
4223
4224 if (ra) {
4225 zend_ssa_phi *phi = ssa->blocks[1].phis;
4226
4227 while (phi) {
4228 zend_lifetime_interval *ival = ra[phi->ssa_var];
4229
4230 if (ival) {
4231 if (ival->flags & ZREG_LOAD) {
4232 uint32_t info = ssa->var_info[phi->ssa_var].type;
4233 ZEND_ASSERT(ival->reg != ZREG_NONE);
4234
4235 if (info & MAY_BE_GUARD) {
4236 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4237 goto jit_failure;
4238 }
4239 info &= ~MAY_BE_GUARD;
4240 ssa->var_info[phi->ssa_var].type = info;
4241 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4242 }
4243 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_LOAD);
4244 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
4245 goto jit_failure;
4246 }
4247 } else if (ival->flags & ZREG_STORE) {
4248 ZEND_ASSERT(ival->reg != ZREG_NONE);
4249
4250 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_STORE);
4251 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg,
4252 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4253 goto jit_failure;
4254 }
4255 } else {
4256 /* Register has to be written back on side exit */
4257 SET_STACK_REG(stack, phi->var, ival->reg);
4258 }
4259 }
4260 phi = phi->next;
4261 }
4262 }
4263
4264 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4265 // if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4266 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4267 //
4268 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4269 // if (!timeout_exit_addr) {
4270 // goto jit_failure;
4271 // }
4272 // }
4273 // }
4274
4275 if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
4276 int last_var = op_array->last_var;
4277
4278 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4279 last_var += op_array->T;
4280 }
4281 for (i = 0; i < last_var; i++) {
4282 if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) {
4283 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4284 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4285 goto jit_failure;
4286 }
4287 }
4288 }
4289 }
4290 }
4291
4292 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4293 for (;;p++) {
4294 if (p->op == ZEND_JIT_TRACE_VM) {
4295 uint8_t op1_type = p->op1_type;
4296 uint8_t op2_type = p->op2_type;
4297 uint8_t op3_type = p->op3_type;
4298 uint8_t orig_op1_type = op1_type;
4299 uint8_t orig_op2_type = op2_type;
4300 uint8_t val_type = IS_UNKNOWN;
4301 bool op1_indirect;
4302 zend_class_entry *op1_ce = NULL;
4303 zend_class_entry *op2_ce = NULL;
4304 bool gen_handler = false;
4305
4306 opline = p->opline;
4307 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4308 op1_type = IS_UNKNOWN;
4309 }
4310 if (op1_type != IS_UNKNOWN) {
4311 op1_type &= ~IS_TRACE_PACKED;
4312 }
4313 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4314 op2_type = IS_UNKNOWN;
4315 }
4316 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4317 op3_type = IS_UNKNOWN;
4318 }
4319
4320 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4321 op1_ce = (zend_class_entry*)(p+1)->ce;
4322 p++;
4323 }
4324 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4325 op2_ce = (zend_class_entry*)(p+1)->ce;
4326 p++;
4327 }
4328 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4329 val_type = (p+1)->op1_type;
4330 p++;
4331 }
4332
4333 frame_flags = 0;
4334
4335 switch (opline->opcode) {
4336 case ZEND_INIT_FCALL:
4337 case ZEND_INIT_FCALL_BY_NAME:
4338 case ZEND_INIT_NS_FCALL_BY_NAME:
4339 case ZEND_INIT_METHOD_CALL:
4340 case ZEND_INIT_DYNAMIC_CALL:
4341 case ZEND_INIT_STATIC_METHOD_CALL:
4342 case ZEND_INIT_USER_CALL:
4343 case ZEND_NEW:
4344 frame->call_level++;
4345 }
4346
4347 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4348 gen_handler = 0;
4349 switch (opline->opcode) {
4350 case ZEND_PRE_INC:
4351 case ZEND_PRE_DEC:
4352 case ZEND_POST_INC:
4353 case ZEND_POST_DEC:
4354 if (opline->op1_type != IS_CV) {
4355 break;
4356 }
4357 op1_info = OP1_INFO();
4358 CHECK_OP1_TRACE_TYPE();
4359 if (!(op1_info & MAY_BE_LONG)) {
4360 break;
4361 }
4362 if (opline->result_type != IS_UNUSED) {
4363 res_use_info = zend_jit_trace_type_to_info(
4364 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4365 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4366 res_info = RES_INFO();
4367 res_addr = RES_REG_ADDR();
4368 } else {
4369 res_use_info = -1;
4370 res_info = -1;
4371 res_addr = 0;
4372 }
4373 op1_def_info = OP1_DEF_INFO();
4374 if (op1_def_info & MAY_BE_GUARD
4375 && !has_concrete_type(op1_def_info)) {
4376 op1_def_info &= ~MAY_BE_GUARD;
4377 }
4378 if (!zend_jit_inc_dec(&dasm_state, opline,
4379 op1_info, OP1_REG_ADDR(),
4380 op1_def_info, OP1_DEF_REG_ADDR(),
4381 res_use_info, res_info,
4382 res_addr,
4383 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4384 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4385 goto jit_failure;
4386 }
4387 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4388 && !(op1_info & MAY_BE_STRING)) {
4389 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4390 if (opline->result_type != IS_UNUSED) {
4391 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4392 }
4393 }
4394 if (opline->result_type != IS_UNUSED
4395 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4396 && !(op1_info & MAY_BE_STRING)) {
4397 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4398 }
4399 goto done;
4400 case ZEND_BW_OR:
4401 case ZEND_BW_AND:
4402 case ZEND_BW_XOR:
4403 case ZEND_SL:
4404 case ZEND_SR:
4405 case ZEND_MOD:
4406 op1_info = OP1_INFO();
4407 CHECK_OP1_TRACE_TYPE();
4408 op2_info = OP2_INFO();
4409 CHECK_OP2_TRACE_TYPE();
4410 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4411 break;
4412 }
4413 if (!(op1_info & MAY_BE_LONG)
4414 || !(op2_info & MAY_BE_LONG)) {
4415 break;
4416 }
4417 res_addr = RES_REG_ADDR();
4418 if (Z_MODE(res_addr) != IS_REG
4419 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4420 send_result = 1;
4421 res_use_info = -1;
4422 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4423 if (!zend_jit_reuse_ip(&dasm_state)) {
4424 goto jit_failure;
4425 }
4426 } else {
4427 res_use_info = zend_jit_trace_type_to_info(
4428 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4429 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4430 }
4431 res_info = RES_INFO();
4432 if (!zend_jit_long_math(&dasm_state, opline,
4433 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4434 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4435 res_use_info, res_info, res_addr,
4436 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4437 goto jit_failure;
4438 }
4439 goto done;
4440 case ZEND_ADD:
4441 case ZEND_SUB:
4442 case ZEND_MUL:
4443 // case ZEND_DIV: // TODO: check for division by zero ???
4444 op1_info = OP1_INFO();
4445 op1_addr = OP1_REG_ADDR();
4446 op2_info = OP2_INFO();
4447 op2_addr = OP2_REG_ADDR();
4448 if (orig_op1_type != IS_UNKNOWN
4449 && (orig_op1_type & IS_TRACE_REFERENCE)
4450 && opline->op1_type == IS_CV
4451 && (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_FCARG1)
4452 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4453 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4454 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4455 goto jit_failure;
4456 }
4457 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4458 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4459 }
4460 } else {
4461 CHECK_OP1_TRACE_TYPE();
4462 }
4463 if (orig_op2_type != IS_UNKNOWN
4464 && (orig_op2_type & IS_TRACE_REFERENCE)
4465 && opline->op2_type == IS_CV
4466 && (Z_MODE(op1_addr) != IS_REG || Z_REG(op1_addr) != ZREG_FCARG1)
4467 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4468 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op2_type, &op2_info, &op2_addr,
4469 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4470 goto jit_failure;
4471 }
4472 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4473 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4474 }
4475 } else {
4476 CHECK_OP2_TRACE_TYPE();
4477 }
4478 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4479 break;
4480 }
4481 if (opline->opcode == ZEND_ADD &&
4482 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4483 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4484 /* pass */
4485 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4486 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4487 break;
4488 }
4489 res_addr = RES_REG_ADDR();
4490 if (Z_MODE(res_addr) != IS_REG
4491 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4492 send_result = 1;
4493 res_use_info = -1;
4494 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4495 if (!zend_jit_reuse_ip(&dasm_state)) {
4496 goto jit_failure;
4497 }
4498 } else {
4499 res_use_info = zend_jit_trace_type_to_info(
4500 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4501 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4502 }
4503 res_info = RES_INFO();
4504 if (opline->opcode == ZEND_ADD &&
4505 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4506 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4507 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4508 goto jit_failure;
4509 }
4510 } else {
4511 if (!zend_jit_math(&dasm_state, opline,
4512 op1_info, op1_addr,
4513 op2_info, op2_addr,
4514 res_use_info, res_info, res_addr,
4515 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4516 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4517 goto jit_failure;
4518 }
4519 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4520 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4521 && has_concrete_type(op1_info)
4522 && has_concrete_type(op2_info)) {
4523 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4524 }
4525 }
4526 goto done;
4527 case ZEND_CONCAT:
4528 case ZEND_FAST_CONCAT:
4529 op1_info = OP1_INFO();
4530 CHECK_OP1_TRACE_TYPE();
4531 op2_info = OP2_INFO();
4532 CHECK_OP2_TRACE_TYPE();
4533 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4534 break;
4535 }
4536 if (!(op1_info & MAY_BE_STRING) ||
4537 !(op2_info & MAY_BE_STRING)) {
4538 break;
4539 }
4540 res_addr = RES_REG_ADDR();
4541 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4542 send_result = 1;
4543 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4544 if (!zend_jit_reuse_ip(&dasm_state)) {
4545 goto jit_failure;
4546 }
4547 }
4548 if (!zend_jit_concat(&dasm_state, opline,
4549 op1_info, op2_info, res_addr,
4550 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4551 goto jit_failure;
4552 }
4553 goto done;
4554 case ZEND_ASSIGN_OP:
4555 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4556 break;
4557 }
4558 op1_info = OP1_INFO();
4559 CHECK_OP1_TRACE_TYPE();
4560 op2_info = OP2_INFO();
4561 CHECK_OP2_TRACE_TYPE();
4562 if (!zend_jit_supported_binary_op(
4563 opline->extended_value, op1_info, op2_info)) {
4564 break;
4565 }
4566 op1_def_info = OP1_DEF_INFO();
4567 if (op1_def_info & MAY_BE_GUARD
4568 && !has_concrete_type(op1_def_info)) {
4569 op1_def_info &= ~MAY_BE_GUARD;
4570 }
4571 if (!zend_jit_assign_op(&dasm_state, opline,
4572 op1_info, op1_def_info, OP1_RANGE(),
4573 op2_info, OP2_RANGE(),
4574 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4575 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4576 goto jit_failure;
4577 }
4578 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4579 && has_concrete_type(op1_info)
4580 && has_concrete_type(op2_info)) {
4581 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4582 if (opline->result_type != IS_UNUSED) {
4583 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4584 }
4585 }
4586 goto done;
4587 case ZEND_ASSIGN_DIM_OP:
4588 if (opline->result_type != IS_UNUSED) {
4589 break;
4590 }
4591 if (!zend_jit_supported_binary_op(
4592 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4593 break;
4594 }
4595 if (opline->op1_type == IS_CV
4596 && (opline+1)->op1_type == IS_CV
4597 && (opline+1)->op1.var == opline->op1.var) {
4598 /* skip $a[x] += $a; */
4599 break;
4600 }
4601 op1_info = OP1_INFO();
4602 op1_addr = OP1_REG_ADDR();
4603 if (opline->op1_type == IS_VAR) {
4604 if (orig_op1_type != IS_UNKNOWN
4605 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4606 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4607 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4608 goto jit_failure;
4609 }
4610 } else {
4611 break;
4612 }
4613 }
4614 if (orig_op1_type != IS_UNKNOWN
4615 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4616 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4617 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4618 goto jit_failure;
4619 }
4620 if (opline->op1_type == IS_CV
4621 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4622 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4623 }
4624 } else {
4625 CHECK_OP1_TRACE_TYPE();
4626 }
4627 op2_info = OP2_INFO();
4628 CHECK_OP2_TRACE_TYPE();
4629 op1_data_info = OP1_DATA_INFO();
4630 CHECK_OP1_DATA_TRACE_TYPE();
4631 op1_def_info = OP1_DEF_INFO();
4632 if (!zend_jit_assign_dim_op(&dasm_state, opline,
4633 op1_info, op1_def_info, op1_addr, op2_info,
4634 op1_data_info, OP1_DATA_RANGE(), val_type,
4635 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4636 goto jit_failure;
4637 }
4638 goto done;
4639 case ZEND_PRE_INC_OBJ:
4640 case ZEND_PRE_DEC_OBJ:
4641 case ZEND_POST_INC_OBJ:
4642 case ZEND_POST_DEC_OBJ:
4643 if (opline->op2_type != IS_CONST
4644 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4645 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4646 break;
4647 }
4648 ce = NULL;
4649 ce_is_instanceof = 0;
4650 on_this = delayed_fetch_this = 0;
4651 op1_indirect = 0;
4652 if (opline->op1_type == IS_UNUSED) {
4653 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4654 ce = op_array->scope;
4655 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4656 op1_addr = 0;
4657 on_this = 1;
4658 } else {
4659 if (ssa_op->op1_use >= 0) {
4660 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4661 }
4662 op1_info = OP1_INFO();
4663 if (!(op1_info & MAY_BE_OBJECT)) {
4664 break;
4665 }
4666 op1_addr = OP1_REG_ADDR();
4667 if (opline->op1_type == IS_VAR) {
4668 if (orig_op1_type != IS_UNKNOWN
4669 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4670 op1_indirect = 1;
4671 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4672 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4673 goto jit_failure;
4674 }
4675 }
4676 }
4677 if (orig_op1_type != IS_UNKNOWN
4678 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4679 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4680 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4681 goto jit_failure;
4682 }
4683 if (opline->op1_type == IS_CV
4684 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4685 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4686 }
4687 } else {
4688 CHECK_OP1_TRACE_TYPE();
4689 }
4690 if (!(op1_info & MAY_BE_OBJECT)) {
4691 break;
4692 }
4693 if (ssa->var_info && ssa->ops) {
4694 if (ssa_op->op1_use >= 0) {
4695 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4696 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4697 ce = op1_ssa->ce;
4698 ce_is_instanceof = op1_ssa->is_instanceof;
4699 }
4700 }
4701 }
4702 if (delayed_fetch_this) {
4703 on_this = 1;
4704 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4705 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4706 } else if (op_array_ssa->ops
4707 && op_array_ssa->vars
4708 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4709 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4710 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4711 }
4712 }
4713 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4714 op1_info, op1_addr,
4715 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4716 val_type)) {
4717 goto jit_failure;
4718 }
4719 goto done;
4720 case ZEND_ASSIGN_OBJ_OP:
4721 if (opline->result_type != IS_UNUSED) {
4722 break;
4723 }
4724 if (opline->op2_type != IS_CONST
4725 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4726 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4727 break;
4728 }
4729 if (opline->op1_type == IS_CV
4730 && (opline+1)->op1_type == IS_CV
4731 && (opline+1)->op1.var == opline->op1.var) {
4732 /* skip $a->prop += $a; */
4733 break;
4734 }
4735 if (!zend_jit_supported_binary_op(
4736 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4737 break;
4738 }
4739 ce = NULL;
4740 ce_is_instanceof = 0;
4741 on_this = delayed_fetch_this = 0;
4742 op1_indirect = 0;
4743 if (opline->op1_type == IS_UNUSED) {
4744 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4745 ce = op_array->scope;
4746 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4747 op1_addr = 0;
4748 on_this = 1;
4749 } else {
4750 if (ssa_op->op1_use >= 0) {
4751 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4752 }
4753 op1_info = OP1_INFO();
4754 if (!(op1_info & MAY_BE_OBJECT)) {
4755 break;
4756 }
4757 op1_addr = OP1_REG_ADDR();
4758 if (opline->op1_type == IS_VAR) {
4759 if (orig_op1_type != IS_UNKNOWN
4760 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4761 op1_indirect = 1;
4762 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4763 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4764 goto jit_failure;
4765 }
4766 }
4767 }
4768 if (orig_op1_type != IS_UNKNOWN
4769 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4770 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4771 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4772 goto jit_failure;
4773 }
4774 if (opline->op1_type == IS_CV
4775 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4776 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4777 }
4778 } else {
4779 CHECK_OP1_TRACE_TYPE();
4780 }
4781 if (!(op1_info & MAY_BE_OBJECT)) {
4782 break;
4783 }
4784 if (ssa->var_info && ssa->ops) {
4785 if (ssa_op->op1_use >= 0) {
4786 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4787 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4788 ce = op1_ssa->ce;
4789 ce_is_instanceof = op1_ssa->is_instanceof;
4790 }
4791 }
4792 }
4793 if (delayed_fetch_this) {
4794 on_this = 1;
4795 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4796 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4797 } else if (op_array_ssa->ops
4798 && op_array_ssa->vars
4799 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4800 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4801 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4802 }
4803 }
4804 op1_data_info = OP1_DATA_INFO();
4805 CHECK_OP1_DATA_TRACE_TYPE();
4806 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
4807 op1_info, op1_addr, op1_data_info, OP1_DATA_RANGE(),
4808 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4809 val_type)) {
4810 goto jit_failure;
4811 }
4812 goto done;
4813 case ZEND_ASSIGN_OBJ:
4814 if (opline->op2_type != IS_CONST
4815 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4816 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4817 break;
4818 }
4819 ce = NULL;
4820 ce_is_instanceof = 0;
4821 on_this = delayed_fetch_this = 0;
4822 op1_indirect = 0;
4823 if (opline->op1_type == IS_UNUSED) {
4824 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4825 ce = op_array->scope;
4826 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4827 op1_addr = 0;
4828 on_this = 1;
4829 } else {
4830 if (ssa_op->op1_use >= 0) {
4831 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4832 }
4833 op1_info = OP1_INFO();
4834 if (!(op1_info & MAY_BE_OBJECT)) {
4835 break;
4836 }
4837 op1_addr = OP1_REG_ADDR();
4838 if (opline->op1_type == IS_VAR) {
4839 if (orig_op1_type != IS_UNKNOWN
4840 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4841 op1_indirect = 1;
4842 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4843 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4844 goto jit_failure;
4845 }
4846 }
4847 }
4848 if (orig_op1_type != IS_UNKNOWN
4849 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4850 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4851 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4852 goto jit_failure;
4853 }
4854 if (opline->op1_type == IS_CV
4855 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4856 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4857 }
4858 } else {
4859 CHECK_OP1_TRACE_TYPE();
4860 }
4861 if (!(op1_info & MAY_BE_OBJECT)) {
4862 break;
4863 }
4864 if (ssa->var_info && ssa->ops) {
4865 if (ssa_op->op1_use >= 0) {
4866 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4867 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4868 ce = op1_ssa->ce;
4869 ce_is_instanceof = op1_ssa->is_instanceof;
4870 }
4871 }
4872 }
4873 if (delayed_fetch_this) {
4874 on_this = 1;
4875 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4876 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4877 } else if (op_array_ssa->ops
4878 && op_array_ssa->vars
4879 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4880 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4881 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4882 }
4883 }
4884 op1_data_info = OP1_DATA_INFO();
4885 CHECK_OP1_DATA_TRACE_TYPE();
4886 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4887 op1_info, op1_addr, op1_data_info,
4888 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4889 val_type,
4890 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4891 goto jit_failure;
4892 }
4893 if ((opline+1)->op1_type == IS_CV
4894 && (ssa_op+1)->op1_def >= 0
4895 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4896 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4897 }
4898 goto done;
4899 case ZEND_ASSIGN_DIM:
4900 op1_info = OP1_INFO();
4901 op1_addr = OP1_REG_ADDR();
4902 if (opline->op1_type == IS_CV
4903 && (opline+1)->op1_type == IS_CV
4904 && (opline+1)->op1.var == opline->op1.var) {
4905 /* skip $a[x] = $a; */
4906 break;
4907 }
4908 if (opline->op1_type == IS_VAR) {
4909 if (orig_op1_type != IS_UNKNOWN
4910 && (orig_op1_type & IS_TRACE_INDIRECT)
4911 && opline->result_type == IS_UNUSED) {
4912 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4913 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4914 goto jit_failure;
4915 }
4916 } else {
4917 break;
4918 }
4919 }
4920 if (orig_op1_type != IS_UNKNOWN
4921 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4922 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4923 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4924 goto jit_failure;
4925 }
4926 if (opline->op1_type == IS_CV
4927 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4928 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4929 }
4930 } else {
4931 CHECK_OP1_TRACE_TYPE();
4932 }
4933 op2_info = OP2_INFO();
4934 CHECK_OP2_TRACE_TYPE();
4935 op1_data_info = OP1_DATA_INFO();
4936 CHECK_OP1_DATA_TRACE_TYPE();
4937 if (!zend_jit_assign_dim(&dasm_state, opline,
4938 op1_info, op1_addr, op2_info, op1_data_info, val_type,
4939 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4940 goto jit_failure;
4941 }
4942 if ((opline+1)->op1_type == IS_CV
4943 && (ssa_op+1)->op1_def >= 0
4944 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4945 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4946 }
4947 goto done;
4948 case ZEND_ASSIGN:
4949 if (opline->op1_type != IS_CV) {
4950 break;
4951 }
4952 op2_addr = OP2_REG_ADDR();
4953 op2_info = OP2_INFO();
4954
4955 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
4956 op2_def_addr = op2_addr;
4957 } else {
4958 op2_def_addr = OP2_DEF_REG_ADDR();
4959 }
4960 CHECK_OP2_TRACE_TYPE();
4961 op1_info = OP1_INFO();
4962 op1_def_info = OP1_DEF_INFO();
4963 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
4964 if (op1_type < IS_STRING
4965 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
4966 if (!zend_jit_scalar_type_guard(&dasm_state, opline, opline->op1.var)) {
4967 goto jit_failure;
4968 }
4969 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
4970 } else {
4971 CHECK_OP1_TRACE_TYPE();
4972 }
4973 }
4974 op1_addr = OP1_REG_ADDR();
4975 op1_def_addr = OP1_DEF_REG_ADDR();
4976 if (Z_MODE(op1_def_addr) != IS_REG &&
4977 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
4978 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
4979 /* type may be not set */
4980 op1_info |= MAY_BE_NULL;
4981 }
4982 if (orig_op1_type != IS_UNKNOWN) {
4983 if (orig_op1_type & IS_TRACE_REFERENCE) {
4984 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4985 !ssa->var_info[ssa_op->op1_use].guarded_reference, 0)) {
4986 goto jit_failure;
4987 }
4988 if (opline->op1_type == IS_CV
4989 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4990 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4991 }
4992 if (opline->result_type == IS_UNUSED) {
4993 res_addr = 0;
4994 } else {
4995 res_addr = RES_REG_ADDR();
4996 if (Z_MODE(res_addr) != IS_REG
4997 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4998 send_result = 1;
4999 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5000 if (!zend_jit_reuse_ip(&dasm_state)) {
5001 goto jit_failure;
5002 }
5003 }
5004 }
5005 if (!zend_jit_assign_to_typed_ref(&dasm_state, opline, opline->op2_type, op2_addr, res_addr, 1)) {
5006 goto jit_failure;
5007 }
5008 op1_def_addr = op1_addr;
5009 op1_def_info &= ~MAY_BE_REF;
5010 } else if (op1_info & MAY_BE_REF) {
5011 if (!zend_jit_noref_guard(&dasm_state, opline, op1_addr)) {
5012 goto jit_failure;
5013 }
5014 op1_info &= ~MAY_BE_REF;
5015 op1_def_info &= ~MAY_BE_REF;
5016 }
5017 }
5018 if (opline->result_type == IS_UNUSED) {
5019 res_addr = 0;
5020 res_info = -1;
5021 } else {
5022 res_addr = RES_REG_ADDR();
5023 res_info = RES_INFO();
5024 if (Z_MODE(res_addr) != IS_REG
5025 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5026 send_result = 1;
5027 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5028 if (!zend_jit_reuse_ip(&dasm_state)) {
5029 goto jit_failure;
5030 }
5031 }
5032 }
5033 if (!zend_jit_assign(&dasm_state, opline,
5034 op1_info, op1_addr,
5035 op1_def_info, op1_def_addr,
5036 op2_info, op2_addr, op2_def_addr,
5037 res_info, res_addr,
5038 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5039 goto jit_failure;
5040 }
5041 if (ssa_op->op2_def >= 0
5042 && Z_MODE(op2_addr) == IS_REG
5043 && ssa->vars[ssa_op->op2_def].no_val) {
5044 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5045 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5046
5047 if (STACK_MEM_TYPE(stack, var_num) != type
5048 && ssa->vars[ssa_op->op2_def].use_chain < 0
5049 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5050 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5051 return 0;
5052 }
5053 SET_STACK_TYPE(stack, var_num, type, 1);
5054 }
5055 }
5056 if (opline->op2_type == IS_CV
5057 && ssa_op->op2_def >= 0
5058 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5059 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5060 }
5061 goto done;
5062 case ZEND_CAST:
5063 if (opline->extended_value != op1_type) {
5064 break;
5065 }
5066 ZEND_FALLTHROUGH;
5067 case ZEND_QM_ASSIGN:
5068 op1_addr = OP1_REG_ADDR();
5069 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5070 op1_def_addr = op1_addr;
5071 } else {
5072 op1_def_addr = OP1_DEF_REG_ADDR();
5073 }
5074 op1_info = OP1_INFO();
5075 CHECK_OP1_TRACE_TYPE();
5076 res_info = RES_INFO();
5077 res_use_info = zend_jit_trace_type_to_info(
5078 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
5079 res_addr = RES_REG_ADDR();
5080 if (Z_MODE(res_addr) != IS_REG &&
5081 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5082 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5083 /* type may be not set */
5084 res_use_info |= MAY_BE_NULL;
5085 }
5086 if (!zend_jit_qm_assign(&dasm_state, opline,
5087 op1_info, op1_addr, op1_def_addr,
5088 res_use_info, res_info, res_addr)) {
5089 goto jit_failure;
5090 }
5091 if (ssa_op->op1_def >= 0
5092 && Z_MODE(op1_addr) == IS_REG
5093 && ssa->vars[ssa_op->op1_def].no_val) {
5094 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5095 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5096
5097 if (STACK_MEM_TYPE(stack, var_num) != type
5098 && ssa->vars[ssa_op->op1_def].use_chain < 0
5099 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5100 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5101 return 0;
5102 }
5103 SET_STACK_TYPE(stack, var_num, type, 1);
5104 }
5105 }
5106 if (opline->op1_type == IS_CV
5107 && ssa_op->op1_def >= 0
5108 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5109 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5110 }
5111 goto done;
5112 case ZEND_INIT_FCALL:
5113 case ZEND_INIT_FCALL_BY_NAME:
5114 case ZEND_INIT_NS_FCALL_BY_NAME:
5115 frame_flags = TRACE_FRAME_MASK_NESTED;
5116 if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5117 goto jit_failure;
5118 }
5119 goto done;
5120 case ZEND_SEND_VAL:
5121 case ZEND_SEND_VAL_EX:
5122 if (opline->op2_type == IS_CONST) {
5123 /* Named parameters not supported in JIT */
5124 break;
5125 }
5126 if (opline->opcode == ZEND_SEND_VAL_EX
5127 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5128 break;
5129 }
5130 op1_info = OP1_INFO();
5131 CHECK_OP1_TRACE_TYPE();
5132 if (!zend_jit_send_val(&dasm_state, opline,
5133 op1_info, OP1_REG_ADDR())) {
5134 goto jit_failure;
5135 }
5136 if (frame->call && frame->call->func) {
5137 if (opline->op1_type == IS_CONST) {
5138 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5139 } else if (op1_type != IS_UNKNOWN) {
5140 if (op1_type == IS_UNDEF) {
5141 op1_type = IS_NULL;
5142 }
5143 zend_jit_trace_send_type(opline, frame->call, op1_type);
5144 }
5145 }
5146 goto done;
5147 case ZEND_SEND_REF:
5148 if (opline->op2_type == IS_CONST) {
5149 /* Named parameters not supported in JIT */
5150 break;
5151 }
5152 op1_info = OP1_INFO();
5153 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
5154 op1_info, 0)) {
5155 goto jit_failure;
5156 }
5157 if (opline->op1_type == IS_CV
5158 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5159 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5160 }
5161 goto done;
5162 case ZEND_SEND_VAR:
5163 case ZEND_SEND_VAR_EX:
5164 case ZEND_SEND_VAR_NO_REF:
5165 case ZEND_SEND_VAR_NO_REF_EX:
5166 case ZEND_SEND_FUNC_ARG:
5167 if (opline->op2_type == IS_CONST) {
5168 /* Named parameters not supported in JIT */
5169 break;
5170 }
5171 if ((opline->opcode == ZEND_SEND_VAR_EX
5172 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5173 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5174 break;
5175 }
5176 op1_addr = OP1_REG_ADDR();
5177 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5178 op1_def_addr = op1_addr;
5179 } else {
5180 op1_def_addr = OP1_DEF_REG_ADDR();
5181 }
5182 op1_info = OP1_INFO();
5183 CHECK_OP1_TRACE_TYPE();
5184 if (!zend_jit_send_var(&dasm_state, opline, op_array,
5185 op1_info, op1_addr, op1_def_addr)) {
5186 goto jit_failure;
5187 }
5188 if (ssa_op->op1_def >= 0
5189 && Z_MODE(op1_addr) == IS_REG
5190 && ssa->vars[ssa_op->op1_def].no_val) {
5191 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5192 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5193
5194 if (STACK_MEM_TYPE(stack, var_num) != type
5195 && ssa->vars[ssa_op->op1_def].use_chain < 0
5196 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5197 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5198 return 0;
5199 }
5200 SET_STACK_TYPE(stack, var_num, type, 1);
5201 }
5202 }
5203 if (opline->op1_type == IS_CV
5204 && ssa_op->op1_def >= 0
5205 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5206 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5207 }
5208 if (frame->call && frame->call->func) {
5209 if ((opline->opcode == ZEND_SEND_VAR_EX
5210 || opline->opcode == ZEND_SEND_FUNC_ARG)
5211 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5212 goto done;
5213 }
5214 if (op1_type != IS_UNKNOWN) {
5215 if (op1_type == IS_UNDEF) {
5216 op1_type = IS_NULL;
5217 }
5218 zend_jit_trace_send_type(opline, frame->call, op1_type);
5219 }
5220 }
5221 goto done;
5222 case ZEND_CHECK_FUNC_ARG:
5223 if (!JIT_G(current_frame)
5224 || !JIT_G(current_frame)->call
5225 || !JIT_G(current_frame)->call->func) {
5226 break;
5227 }
5228 if (opline->op2_type == IS_CONST
5229 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5230 /* Named parameters not supported in JIT */
5231 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5232 break;
5233 }
5234 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
5235 goto jit_failure;
5236 }
5237 goto done;
5238 case ZEND_CHECK_UNDEF_ARGS:
5239 if (JIT_G(current_frame)
5240 && JIT_G(current_frame)->call) {
5241 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5242 }
5243 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
5244 goto jit_failure;
5245 }
5246 goto done;
5247 case ZEND_DO_UCALL:
5248 case ZEND_DO_ICALL:
5249 case ZEND_DO_FCALL_BY_NAME:
5250 case ZEND_DO_FCALL:
5251 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5252 goto jit_failure;
5253 }
5254 goto done;
5255 case ZEND_IS_EQUAL:
5256 case ZEND_IS_NOT_EQUAL:
5257 case ZEND_IS_SMALLER:
5258 case ZEND_IS_SMALLER_OR_EQUAL:
5259 case ZEND_CASE:
5260 op1_info = OP1_INFO();
5261 op2_info = OP2_INFO();
5262 skip_comparison =
5263 ssa_op != ssa->ops &&
5264 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5265 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5266 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5267 CHECK_OP1_TRACE_TYPE();
5268 CHECK_OP2_TRACE_TYPE();
5269 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5270 bool exit_if_true = 0;
5271 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5272 uint32_t exit_point;
5273
5274 if (ra) {
5275 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5276 }
5277 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5278 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5279 if (!exit_addr) {
5280 goto jit_failure;
5281 }
5282 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5283 if (!zend_jit_cmp(&dasm_state, opline,
5284 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5285 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5286 RES_REG_ADDR(),
5287 zend_may_throw(opline, ssa_op, op_array, ssa),
5288 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5289 goto jit_failure;
5290 }
5291 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5292 } else {
5293 smart_branch_opcode = 0;
5294 exit_addr = NULL;
5295 if (!zend_jit_cmp(&dasm_state, opline,
5296 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5297 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5298 RES_REG_ADDR(),
5299 zend_may_throw(opline, ssa_op, op_array, ssa),
5300 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5301 goto jit_failure;
5302 }
5303 }
5304 goto done;
5305 case ZEND_IS_IDENTICAL:
5306 case ZEND_IS_NOT_IDENTICAL:
5307 case ZEND_CASE_STRICT:
5308 op1_info = OP1_INFO();
5309 op2_info = OP2_INFO();
5310 skip_comparison =
5311 ssa_op != ssa->ops &&
5312 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5313 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5314 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5315 CHECK_OP1_TRACE_TYPE();
5316 CHECK_OP2_TRACE_TYPE();
5317 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5318 bool exit_if_true = 0;
5319 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5320 uint32_t exit_point;
5321
5322 if (ra) {
5323 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5324 }
5325 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5326 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5327 if (!exit_addr) {
5328 goto jit_failure;
5329 }
5330 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5331 exit_if_true = !exit_if_true;
5332 }
5333 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5334 if (!zend_jit_identical(&dasm_state, opline,
5335 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5336 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5337 RES_REG_ADDR(),
5338 zend_may_throw(opline, ssa_op, op_array, ssa),
5339 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5340 goto jit_failure;
5341 }
5342 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5343 } else {
5344 smart_branch_opcode = 0;
5345 exit_addr = NULL;
5346 if (!zend_jit_identical(&dasm_state, opline,
5347 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5348 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5349 RES_REG_ADDR(),
5350 zend_may_throw(opline, ssa_op, op_array, ssa),
5351 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5352 goto jit_failure;
5353 }
5354 }
5355 goto done;
5356 case ZEND_DEFINED:
5357 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5358 bool exit_if_true = 0;
5359 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5360 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5361
5362 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5363 if (!exit_addr) {
5364 goto jit_failure;
5365 }
5366 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5367 } else {
5368 smart_branch_opcode = 0;
5369 exit_addr = NULL;
5370 }
5371 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5372 goto jit_failure;
5373 }
5374 goto done;
5375 case ZEND_TYPE_CHECK:
5376 if (opline->extended_value == MAY_BE_RESOURCE) {
5377 // TODO: support for is_resource() ???
5378 break;
5379 }
5380 op1_info = OP1_INFO();
5381 CHECK_OP1_TRACE_TYPE();
5382 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5383 bool exit_if_true = 0;
5384 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5385 uint32_t exit_point;
5386
5387 if (ra) {
5388 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5389 }
5390 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5391 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5392 if (!exit_addr) {
5393 goto jit_failure;
5394 }
5395 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5396 } else {
5397 smart_branch_opcode = 0;
5398 exit_addr = NULL;
5399 }
5400 if (!zend_jit_type_check(&dasm_state, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5401 goto jit_failure;
5402 }
5403 goto done;
5404 case ZEND_RETURN:
5405 op1_info = OP1_INFO();
5406 CHECK_OP1_TRACE_TYPE();
5407 if (opline->op1_type == IS_CONST) {
5408 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5409 } else if (op1_type != IS_UNKNOWN) {
5410 res_type = op1_type;
5411 if (res_type == IS_UNDEF) {
5412 res_type = IS_NULL;
5413 }
5414 }
5415 if (op_array->type == ZEND_EVAL_CODE
5416 // TODO: support for top-level code
5417 || !op_array->function_name
5418 // TODO: support for IS_UNDEF ???
5419 || (op1_info & MAY_BE_UNDEF)) {
5420 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5421 goto jit_failure;
5422 }
5423 } else {
5424 int j;
5425 int may_throw = 0;
5426 bool left_frame = 0;
5427
5428 if (!zend_jit_return(&dasm_state, opline, op_array,
5429 op1_info, OP1_REG_ADDR())) {
5430 goto jit_failure;
5431 }
5432 if (op_array->last_var > 100) {
5433 /* To many CVs to unroll */
5434 if (!zend_jit_free_cvs(&dasm_state)) {
5435 goto jit_failure;
5436 }
5437 left_frame = 1;
5438 }
5439 if (!left_frame) {
5440 for (j = 0 ; j < op_array->last_var; j++) {
5441 uint32_t info;
5442 uint8_t type;
5443
5444 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5445 type = STACK_TYPE(stack, j);
5446 info = zend_jit_trace_type_to_info_ex(type, info);
5447 if (opline->op1_type == IS_CV
5448 && EX_VAR_TO_NUM(opline->op1.var) == j
5449 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5450 if (JIT_G(current_frame)
5451 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5452 continue;
5453 } else {
5454 info |= MAY_BE_NULL;
5455 }
5456 }
5457 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5458 if (!left_frame) {
5459 left_frame = 1;
5460 if (!zend_jit_leave_frame(&dasm_state)) {
5461 goto jit_failure;
5462 }
5463 }
5464 if (!zend_jit_free_cv(&dasm_state, info, j)) {
5465 goto jit_failure;
5466 }
5467 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5468 if (info & MAY_BE_RC1) {
5469 may_throw = 1;
5470 }
5471 }
5472 }
5473 }
5474 }
5475 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
5476 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5477 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5478 goto jit_failure;
5479 }
5480 }
5481 goto done;
5482 case ZEND_BOOL:
5483 case ZEND_BOOL_NOT:
5484 op1_info = OP1_INFO();
5485 CHECK_OP1_TRACE_TYPE();
5486 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5487 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5488 -1, -1,
5489 zend_may_throw(opline, ssa_op, op_array, ssa),
5490 opline->opcode, NULL)) {
5491 goto jit_failure;
5492 }
5493 goto done;
5494 case ZEND_JMPZ:
5495 case ZEND_JMPNZ:
5496 case ZEND_JMPZ_EX:
5497 case ZEND_JMPNZ_EX:
5498 op1_info = OP1_INFO();
5499 CHECK_OP1_TRACE_TYPE();
5500 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5501 const zend_op *exit_opline = NULL;
5502 uint32_t exit_point;
5503
5504 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5505 /* taken branch */
5506 if (opline->opcode == ZEND_JMPNZ_EX) {
5507 smart_branch_opcode = ZEND_JMPZ_EX;
5508 } else if (opline->opcode == ZEND_JMPZ_EX) {
5509 smart_branch_opcode = ZEND_JMPNZ_EX;
5510 } else if (opline->opcode == ZEND_JMPNZ) {
5511 smart_branch_opcode = ZEND_JMPZ;
5512 } else {
5513 smart_branch_opcode = ZEND_JMPNZ;
5514 }
5515 exit_opline = opline + 1;
5516 } else if ((p+1)->opline == opline + 1) {
5517 /* not taken branch */
5518 smart_branch_opcode = opline->opcode;
5519 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5520 } else {
5521 ZEND_UNREACHABLE();
5522 }
5523 if (ra) {
5524 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5525 }
5526 if (!(op1_info & MAY_BE_GUARD)
5527 && has_concrete_type(op1_info)
5528 && concrete_type(op1_info) <= IS_TRUE) {
5529 /* unconditional branch */
5530 exit_addr = NULL;
5531 } else if (opline->result_type == IS_TMP_VAR) {
5532 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5533 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5534
5535 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5536 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5537 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5538 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5539 if (!exit_addr) {
5540 goto jit_failure;
5541 }
5542 } else {
5543 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5544 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5545 if (!exit_addr) {
5546 goto jit_failure;
5547 }
5548 }
5549 } else {
5550 ZEND_UNREACHABLE();
5551 }
5552 if (opline->result_type == IS_UNDEF) {
5553 res_addr = 0;
5554 } else {
5555 res_addr = RES_REG_ADDR();
5556 }
5557 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5558 op1_info, OP1_REG_ADDR(), res_addr,
5559 -1, -1,
5560 zend_may_throw(opline, ssa_op, op_array, ssa),
5561 smart_branch_opcode, exit_addr)) {
5562 goto jit_failure;
5563 }
5564 goto done;
5565 case ZEND_ISSET_ISEMPTY_CV:
5566 if ((opline->extended_value & ZEND_ISEMPTY)) {
5567 // TODO: support for empty() ???
5568 break;
5569 }
5570 op1_info = OP1_INFO();
5571 op1_addr = OP1_REG_ADDR();
5572 if (orig_op1_type != IS_UNKNOWN
5573 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5574 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5575 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5576 goto jit_failure;
5577 }
5578 if (opline->op1_type == IS_CV
5579 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5580 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5581 }
5582 } else {
5583 CHECK_OP1_TRACE_TYPE();
5584 }
5585 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5586 bool exit_if_true = 0;
5587 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5588 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5589
5590 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5591 if (!exit_addr) {
5592 goto jit_failure;
5593 }
5594 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5595 } else {
5596 smart_branch_opcode = 0;
5597 exit_addr = NULL;
5598 }
5599 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
5600 op1_info, op1_addr,
5601 smart_branch_opcode, -1, -1, exit_addr)) {
5602 goto jit_failure;
5603 }
5604 goto done;
5605 case ZEND_IN_ARRAY:
5606 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5607 break;
5608 }
5609 op1_info = OP1_INFO();
5610 op1_addr = OP1_REG_ADDR();
5611 CHECK_OP1_TRACE_TYPE();
5612 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5613 break;
5614 }
5615 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5616 bool exit_if_true = 0;
5617 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5618 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5619
5620 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5621 if (!exit_addr) {
5622 goto jit_failure;
5623 }
5624 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5625 } else {
5626 smart_branch_opcode = 0;
5627 exit_addr = NULL;
5628 }
5629 if (!zend_jit_in_array(&dasm_state, opline,
5630 op1_info, op1_addr,
5631 smart_branch_opcode, -1, -1, exit_addr)) {
5632 goto jit_failure;
5633 }
5634 goto done;
5635 case ZEND_FETCH_DIM_FUNC_ARG:
5636 if (!JIT_G(current_frame)
5637 || !JIT_G(current_frame)->call
5638 || !JIT_G(current_frame)->call->func
5639 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5640 break;
5641 }
5642 ZEND_FALLTHROUGH;
5643 case ZEND_FETCH_DIM_R:
5644 case ZEND_FETCH_DIM_IS:
5645 case ZEND_FETCH_LIST_R:
5646 op1_info = OP1_INFO();
5647 op1_addr = OP1_REG_ADDR();
5648 if (orig_op1_type != IS_UNKNOWN
5649 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5650 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5651 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5652 goto jit_failure;
5653 }
5654 if (opline->op1_type == IS_CV
5655 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5656 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5657 if (ssa_op->op1_def >= 0) {
5658 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5659 }
5660 }
5661 } else {
5662 CHECK_OP1_TRACE_TYPE();
5663 }
5664 op2_info = OP2_INFO();
5665 CHECK_OP2_TRACE_TYPE();
5666 res_info = RES_INFO();
5667 avoid_refcounting =
5668 ssa_op->op1_use >= 0 &&
5669 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5670 if (op1_info & MAY_BE_PACKED_GUARD) {
5671 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5672 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5673 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5674 && MAY_BE_PACKED(op1_info)
5675 && MAY_BE_HASH(op1_info)
5676 && orig_op1_type != IS_UNKNOWN) {
5677 op1_info |= MAY_BE_PACKED_GUARD;
5678 if (orig_op1_type & IS_TRACE_PACKED) {
5679 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5680 if (op1_type != IS_UNKNOWN) {
5681 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5682 }
5683 } else {
5684 op1_info &= ~MAY_BE_ARRAY_PACKED;
5685 if (op1_type != IS_UNKNOWN) {
5686 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5687 }
5688 }
5689 }
5690 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
5691 op1_info, op1_addr, avoid_refcounting,
5692 op2_info, res_info, RES_REG_ADDR(), val_type)) {
5693 goto jit_failure;
5694 }
5695 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5696 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5697 }
5698 goto done;
5699 case ZEND_FETCH_DIM_W:
5700 case ZEND_FETCH_DIM_RW:
5701 // case ZEND_FETCH_DIM_UNSET:
5702 case ZEND_FETCH_LIST_W:
5703 if (opline->op1_type != IS_CV
5704 && (orig_op1_type == IS_UNKNOWN
5705 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5706 break;
5707 }
5708 op1_info = OP1_INFO();
5709 op1_addr = OP1_REG_ADDR();
5710 if (opline->op1_type == IS_VAR) {
5711 if (orig_op1_type != IS_UNKNOWN
5712 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5713 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5714 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5715 goto jit_failure;
5716 }
5717 } else {
5718 break;
5719 }
5720 }
5721 if (orig_op1_type != IS_UNKNOWN
5722 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5723 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5724 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5725 goto jit_failure;
5726 }
5727 if (opline->op1_type == IS_CV
5728 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5729 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5730 }
5731 } else {
5732 CHECK_OP1_TRACE_TYPE();
5733 }
5734 op2_info = OP2_INFO();
5735 CHECK_OP2_TRACE_TYPE();
5736 op1_def_info = OP1_DEF_INFO();
5737 if (!zend_jit_fetch_dim(&dasm_state, opline,
5738 op1_info, op1_addr, op2_info, RES_REG_ADDR(), val_type)) {
5739 goto jit_failure;
5740 }
5741 if (ssa_op->result_def >= 0
5742 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5743 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5744 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5745 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5746 }
5747 goto done;
5748 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5749 if ((opline->extended_value & ZEND_ISEMPTY)) {
5750 // TODO: support for empty() ???
5751 break;
5752 }
5753 op1_info = OP1_INFO();
5754 op1_addr = OP1_REG_ADDR();
5755 if (orig_op1_type != IS_UNKNOWN
5756 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5757 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5758 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5759 goto jit_failure;
5760 }
5761 if (opline->op1_type == IS_CV
5762 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5763 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5764 }
5765 } else {
5766 CHECK_OP1_TRACE_TYPE();
5767 }
5768 op2_info = OP2_INFO();
5769 CHECK_OP2_TRACE_TYPE();
5770 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5771 bool exit_if_true = 0;
5772 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5773 uint32_t exit_point;
5774
5775 if (ra) {
5776 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5777 }
5778 if (ssa_op->op1_use >= 0
5779 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5780 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5781 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5782 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5783
5784 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5785 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5786 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5787 } else {
5788 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5789 }
5790 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5791 if (!exit_addr) {
5792 goto jit_failure;
5793 }
5794 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5795 } else {
5796 smart_branch_opcode = 0;
5797 exit_addr = NULL;
5798 }
5799 avoid_refcounting =
5800 ssa_op->op1_use >= 0 &&
5801 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5802 if (op1_info & MAY_BE_PACKED_GUARD) {
5803 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5804 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5805 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5806 && MAY_BE_PACKED(op1_info)
5807 && MAY_BE_HASH(op1_info)
5808 && orig_op1_type != IS_UNKNOWN) {
5809 op1_info |= MAY_BE_PACKED_GUARD;
5810 if (orig_op1_type & IS_TRACE_PACKED) {
5811 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5812 } else {
5813 op1_info &= ~MAY_BE_ARRAY_PACKED;
5814 }
5815 }
5816 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
5817 op1_info, op1_addr, avoid_refcounting,
5818 op2_info, val_type,
5819 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5820 smart_branch_opcode, -1, -1,
5821 exit_addr)) {
5822 goto jit_failure;
5823 }
5824 goto done;
5825 case ZEND_FETCH_OBJ_FUNC_ARG:
5826 if (!JIT_G(current_frame)
5827 || !JIT_G(current_frame)->call
5828 || !JIT_G(current_frame)->call->func
5829 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5830 break;
5831 }
5832 ZEND_FALLTHROUGH;
5833 case ZEND_FETCH_OBJ_R:
5834 case ZEND_FETCH_OBJ_IS:
5835 case ZEND_FETCH_OBJ_W:
5836 on_this = delayed_fetch_this = 0;
5837 avoid_refcounting = 0;
5838 if (opline->op2_type != IS_CONST
5839 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5840 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5841 break;
5842 }
5843 ce = NULL;
5844 ce_is_instanceof = 0;
5845 op1_indirect = 0;
5846 if (opline->op1_type == IS_UNUSED) {
5847 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
5848 ce = op_array->scope;
5849 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
5850 op1_addr = 0;
5851 on_this = 1;
5852 } else {
5853 op1_info = OP1_INFO();
5854 if (!(op1_info & MAY_BE_OBJECT)) {
5855 break;
5856 }
5857 op1_addr = OP1_REG_ADDR();
5858 if (opline->op1_type == IS_VAR
5859 && opline->opcode == ZEND_FETCH_OBJ_W) {
5860 if (orig_op1_type != IS_UNKNOWN
5861 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5862 op1_indirect = 1;
5863 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5864 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5865 goto jit_failure;
5866 }
5867 }
5868 }
5869 if (orig_op1_type != IS_UNKNOWN
5870 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5871 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5872 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5873 goto jit_failure;
5874 }
5875 if (opline->op1_type == IS_CV
5876 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5877 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
5878 }
5879 } else {
5880 CHECK_OP1_TRACE_TYPE();
5881 }
5882 if (!(op1_info & MAY_BE_OBJECT)) {
5883 break;
5884 }
5885 if (ssa->var_info && ssa->ops) {
5886 if (ssa_op->op1_use >= 0) {
5887 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5888 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5889 ce = op1_ssa->ce;
5890 ce_is_instanceof = op1_ssa->is_instanceof;
5891 }
5892 }
5893 }
5894 if (ssa_op->op1_use >= 0) {
5895 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
5896 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5897 }
5898 if (delayed_fetch_this) {
5899 on_this = 1;
5900 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
5901 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
5902 } else if (op_array_ssa->ops
5903 && op_array_ssa->vars
5904 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
5905 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
5906 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
5907 }
5908 }
5909 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
5910 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
5911 on_this, delayed_fetch_this, avoid_refcounting, op1_ce, val_type,
5912 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
5913 goto jit_failure;
5914 }
5915 goto done;
5916 case ZEND_BIND_GLOBAL:
5917 orig_opline = opline;
5918 orig_ssa_op = ssa_op;
5919 while (1) {
5920 if (!ssa->ops || !ssa->var_info) {
5921 op1_info = MAY_BE_ANY|MAY_BE_REF;
5922 } else {
5923 op1_info = OP1_INFO();
5924 }
5925 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5926 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5927 }
5928 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
5929 goto jit_failure;
5930 }
5931 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
5932 opline++;
5933 ssa_op++;
5934 } else {
5935 break;
5936 }
5937 }
5938 opline = orig_opline;
5939 ssa_op = orig_ssa_op;
5940 goto done;
5941 case ZEND_RECV:
5942 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
5943 goto jit_failure;
5944 }
5945 goto done;
5946 case ZEND_RECV_INIT:
5947 orig_opline = opline;
5948 orig_ssa_op = ssa_op;
5949 while (1) {
5950 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
5951 (opline + 1)->opcode != ZEND_RECV_INIT,
5952 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5953 goto jit_failure;
5954 }
5955 if ((opline+1)->opcode == ZEND_RECV_INIT) {
5956 opline++;
5957 ssa_op++;
5958 } else {
5959 break;
5960 }
5961 }
5962 opline = orig_opline;
5963 ssa_op = orig_ssa_op;
5964 goto done;
5965 case ZEND_FREE:
5966 case ZEND_FE_FREE:
5967 op1_info = OP1_INFO();
5968 if (!zend_jit_free(&dasm_state, opline, op1_info,
5969 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5970 goto jit_failure;
5971 }
5972 goto done;
5973 case ZEND_ECHO:
5974 op1_info = OP1_INFO();
5975 CHECK_OP1_TRACE_TYPE();
5976 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5977 break;
5978 }
5979 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
5980 goto jit_failure;
5981 }
5982 goto done;
5983 case ZEND_STRLEN:
5984 op1_info = OP1_INFO();
5985 op1_addr = OP1_REG_ADDR();
5986 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
5987 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5988 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5989 goto jit_failure;
5990 }
5991 if (opline->op1_type == IS_CV
5992 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5993 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5994 }
5995 } else {
5996 CHECK_OP1_TRACE_TYPE();
5997 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5998 break;
5999 }
6000 }
6001 if (!zend_jit_strlen(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR())) {
6002 goto jit_failure;
6003 }
6004 goto done;
6005 case ZEND_COUNT:
6006 op1_info = OP1_INFO();
6007 op1_addr = OP1_REG_ADDR();
6008 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
6009 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6010 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6011 goto jit_failure;
6012 }
6013 if (opline->op1_type == IS_CV
6014 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6015 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6016 }
6017 } else {
6018 CHECK_OP1_TRACE_TYPE();
6019 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6020 break;
6021 }
6022 }
6023 if (!zend_jit_count(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6024 goto jit_failure;
6025 }
6026 goto done;
6027 case ZEND_FETCH_THIS:
6028 delayed_fetch_this = 0;
6029 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6030 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6031 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6032 delayed_fetch_this = 1;
6033 }
6034 }
6035 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) {
6036 goto jit_failure;
6037 }
6038 goto done;
6039 case ZEND_SWITCH_LONG:
6040 case ZEND_SWITCH_STRING:
6041 case ZEND_MATCH:
6042 if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6043 goto jit_failure;
6044 }
6045 goto done;
6046 case ZEND_VERIFY_RETURN_TYPE:
6047 if (opline->op1_type == IS_UNUSED) {
6048 /* Always throws */
6049 break;
6050 }
6051 if (opline->op1_type == IS_CONST) {
6052 /* TODO Different instruction format, has return value */
6053 break;
6054 }
6055 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6056 /* Not worth bothering with */
6057 break;
6058 }
6059 op1_info = OP1_INFO();
6060 CHECK_OP1_TRACE_TYPE();
6061 if (op1_info & MAY_BE_REF) {
6062 /* TODO May need reference unwrapping. */
6063 break;
6064 }
6065 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, op1_info)) {
6066 goto jit_failure;
6067 }
6068 goto done;
6069 case ZEND_FE_RESET_R:
6070 op1_info = OP1_INFO();
6071 CHECK_OP1_TRACE_TYPE();
6072 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6073 break;
6074 }
6075 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
6076 goto jit_failure;
6077 }
6078 goto done;
6079 case ZEND_FE_FETCH_R:
6080 op1_info = OP1_INFO();
6081 CHECK_OP1_TRACE_TYPE();
6082 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6083 break;
6084 }
6085 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6086 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6087 uint32_t exit_point;
6088
6089 if ((p+1)->opline == exit_opline) {
6090 /* taken branch (exit from loop) */
6091 exit_opline = opline;
6092 smart_branch_opcode = ZEND_NOP;
6093 } else if ((p+1)->opline == opline + 1) {
6094 /* not taken branch (loop) */
6095 smart_branch_opcode = ZEND_JMP;
6096 } else {
6097 ZEND_UNREACHABLE();
6098 }
6099 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6100 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6101 if (!exit_addr) {
6102 goto jit_failure;
6103 }
6104 } else {
6105 ZEND_UNREACHABLE();
6106 }
6107 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
6108 -1, smart_branch_opcode, exit_addr)) {
6109 goto jit_failure;
6110 }
6111 goto done;
6112 case ZEND_FETCH_CONSTANT:
6113 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6114 goto jit_failure;
6115 }
6116 goto done;
6117 case ZEND_INIT_METHOD_CALL:
6118 if (opline->op2_type != IS_CONST
6119 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6120 break;
6121 }
6122 on_this = delayed_fetch_this = 0;
6123 ce = NULL;
6124 ce_is_instanceof = 0;
6125 if (opline->op1_type == IS_UNUSED) {
6126 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6127 ce = op_array->scope;
6128 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
6129 op1_addr = 0;
6130 on_this = 1;
6131 } else {
6132 op1_info = OP1_INFO();
6133 op1_addr = OP1_REG_ADDR();
6134 if (polymorphic_side_trace) {
6135 op1_info = MAY_BE_OBJECT;
6136 op1_addr = 0;
6137 } else if (orig_op1_type != IS_UNKNOWN
6138 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6139 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6140 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6141 goto jit_failure;
6142 }
6143 if (opline->op1_type == IS_CV
6144 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6145 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6146 }
6147 } else {
6148 CHECK_OP1_TRACE_TYPE();
6149 }
6150 if (ssa->var_info && ssa->ops) {
6151 if (ssa_op->op1_use >= 0) {
6152 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6153 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6154 ce = op1_ssa->ce;
6155 ce_is_instanceof = op1_ssa->is_instanceof;
6156 }
6157 }
6158 }
6159 if (ssa_op->op1_use >= 0) {
6160 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6161 }
6162 if (delayed_fetch_this) {
6163 on_this = 1;
6164 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6165 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6166 } else if (op_array_ssa->ops
6167 && op_array_ssa->vars
6168 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6169 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6170 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6171 }
6172 }
6173 frame_flags = TRACE_FRAME_MASK_NESTED;
6174 if (!zend_jit_init_method_call(&dasm_state, opline,
6175 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6176 op_array, ssa, ssa_op, frame->call_level,
6177 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6178 p + 1, peek_checked_stack - checked_stack, polymorphic_side_trace)) {
6179 goto jit_failure;
6180 }
6181 goto done;
6182 case ZEND_INIT_DYNAMIC_CALL:
6183 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6184 break;
6185 }
6186 op2_info = OP2_INFO();
6187 CHECK_OP2_TRACE_TYPE();
6188 frame_flags = TRACE_FRAME_MASK_NESTED;
6189 if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6190 goto jit_failure;
6191 }
6192 goto done;
6193 case ZEND_SEND_ARRAY:
6194 case ZEND_SEND_UNPACK:
6195 if (JIT_G(current_frame)
6196 && JIT_G(current_frame)->call) {
6197 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6198 }
6199 break;
6200 case ZEND_ROPE_INIT:
6201 case ZEND_ROPE_ADD:
6202 case ZEND_ROPE_END:
6203 op2_info = OP2_INFO();
6204 CHECK_OP2_TRACE_TYPE();
6205 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6206 break;
6207 }
6208 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
6209 goto jit_failure;
6210 }
6211 goto done;
6212 default:
6213 break;
6214 }
6215 }
6216
6217 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6218 gen_handler = 1;
6219 op1_info = OP1_INFO();
6220 op2_info = OP2_INFO();
6221 if (op1_info & MAY_BE_GUARD) {
6222 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6223 }
6224 if (op2_info & MAY_BE_GUARD) {
6225 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6226 }
6227 if (!zend_jit_trace_handler(&dasm_state, op_array, opline,
6228 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6229 goto jit_failure;
6230 }
6231 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
6232 if (opline->opcode == ZEND_NEW && opline->result_type != IS_UNUSED) {
6233 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6234 }
6235 if (zend_jit_may_be_polymorphic_call(opline) ||
6236 zend_jit_may_be_modified((p+1)->func, op_array)) {
6237 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
6238 goto jit_failure;
6239 }
6240 }
6241 }
6242 }
6243
6244 done:
6245 polymorphic_side_trace = 0;
6246 switch (opline->opcode) {
6247 case ZEND_DO_FCALL:
6248 case ZEND_DO_ICALL:
6249 case ZEND_DO_UCALL:
6250 case ZEND_DO_FCALL_BY_NAME:
6251 case ZEND_CALLABLE_CONVERT:
6252 frame->call_level--;
6253 }
6254
6255 if (ra) {
6256 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
6257 }
6258
6259 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6260 && STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var)) > ZREG_NUM) {
6261 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6262 }
6263
6264 if (opline->opcode == ZEND_ROPE_INIT) {
6265 /* clear stack slots used by rope */
6266 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6267 uint32_t count =
6268 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6269
6270 do {
6271 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6272 var++;
6273 count--;
6274 } while (count);
6275 }
6276
6277 if (ssa_op) {
6278 zend_ssa_range tmp;
6279
6280 /* Keep information about known types on abstract stack */
6281 if (ssa_op->result_def >= 0) {
6282 uint8_t type = IS_UNKNOWN;
6283
6284 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6285 || send_result) {
6286 /* we didn't set result variable */
6287 type = IS_UNKNOWN;
6288 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6289 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6290 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6291 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6292 if (opline->op1_type != IS_CONST) {
6293 /* copy */
6294 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6295 }
6296 } else if (opline->opcode == ZEND_ASSIGN) {
6297 if (opline->op2_type != IS_CONST
6298 && ssa_op->op1_use >= 0
6299 /* assignment to typed reference may cause conversion */
6300 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6301 /* copy */
6302 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6303 }
6304 } else if (opline->opcode == ZEND_POST_INC
6305 || opline->opcode == ZEND_POST_DEC) {
6306 /* copy */
6307 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6308 }
6309 if (opline->opcode == ZEND_JMP_SET
6310 || opline->opcode == ZEND_COALESCE
6311 || opline->opcode == ZEND_JMP_NULL) {
6312 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6313 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6314 } else if ((p+1)->opline != (opline + 1)) {
6315 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6316 }
6317 } else {
6318 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6319 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->result_def]));
6320 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6321 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6322 }
6323 if (type != IS_UNKNOWN) {
6324 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6325 if (opline->opcode == ZEND_FETCH_THIS
6326 && delayed_fetch_this) {
6327 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_THIS);
6328 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6329 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_TRY_ADDREF);
6330 } else if (ra && ra[ssa_op->result_def]) {
6331 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6332 ra[ssa_op->result_def]->flags & ZREG_STORE);
6333 }
6334 }
6335 }
6336
6337 if (type == IS_LONG
6338 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) {
6339 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6340 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6341 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6342 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6343 ssa->var_info[ssa_op->result_def].has_range = 1;
6344 }
6345 }
6346 if (ssa_op->op1_def >= 0
6347 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6348 || opline->result_type != IS_CV
6349 || opline->result.var != opline->op1.var)) {
6350 uint8_t type = IS_UNKNOWN;
6351
6352 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6353 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6354 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6355 } else if (opline->opcode == ZEND_ASSIGN) {
6356 if (!(OP1_INFO() & MAY_BE_REF)
6357 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6358 if (opline->op2_type != IS_CONST) {
6359 /* copy */
6360 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6361 }
6362 }
6363 } else if (opline->opcode == ZEND_SEND_VAR
6364 || opline->opcode == ZEND_CAST
6365 || opline->opcode == ZEND_QM_ASSIGN
6366 || opline->opcode == ZEND_JMP_SET
6367 || opline->opcode == ZEND_COALESCE
6368 || opline->opcode == ZEND_JMP_NULL
6369 || opline->opcode == ZEND_FE_RESET_R) {
6370 /* keep old value */
6371 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6372 }
6373 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6374 (gen_handler || type == IS_UNKNOWN || !ra ||
6375 (!ra[ssa_op->op1_def] &&
6376 !(ssa->vars[ssa_op->op1_def].no_val &&
6377 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6378 (opline->opcode == ZEND_QM_ASSIGN ||
6379 opline->opcode == ZEND_SEND_VAR ||
6380 opline->opcode == ZEND_SEND_VAR_EX ||
6381 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6382 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6383 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6384 if (type != IS_UNKNOWN) {
6385 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6386 if (ra && ra[ssa_op->op1_def]) {
6387 uint8_t flags = ra[ssa_op->op1_def]->flags & ZREG_STORE;
6388
6389 if (ssa_op->op1_use >= 0) {
6390 if (opline->opcode == ZEND_SEND_VAR
6391 || opline->opcode == ZEND_CAST
6392 || opline->opcode == ZEND_QM_ASSIGN
6393 || opline->opcode == ZEND_JMP_SET
6394 || opline->opcode == ZEND_COALESCE
6395 || opline->opcode == ZEND_JMP_NULL
6396 || opline->opcode == ZEND_FE_RESET_R) {
6397 if (!ra[ssa_op->op1_use]) {
6398 flags |= ZREG_LOAD;
6399 }
6400 }
6401 }
6402 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, flags);
6403 }
6404 }
6405 if (type == IS_LONG
6406 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6407 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6408 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6409 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6410 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6411 ssa->var_info[ssa_op->op1_def].has_range = 1;
6412 }
6413 }
6414 if (ssa_op->op2_def >= 0
6415 && (opline->opcode != ZEND_ASSIGN
6416 || opline->op1_type != IS_CV
6417 || opline->op1.var != opline->op2.var)) {
6418 uint8_t type = IS_UNKNOWN;
6419
6420 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6421 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6422 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6423 } else if (opline->opcode == ZEND_ASSIGN) {
6424 /* keep old value */
6425 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6426 }
6427 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6428 (gen_handler || type == IS_UNKNOWN || !ra ||
6429 (!ra[ssa_op->op2_def] &&
6430 !(ssa->vars[ssa_op->op2_def].no_val &&
6431 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6432 opline->opcode == ZEND_ASSIGN))));
6433 if (type != IS_UNKNOWN) {
6434 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6435 if (ra && ra[ssa_op->op2_def]) {
6436 uint8_t flags = ra[ssa_op->op2_def]->flags & ZREG_STORE;
6437
6438 if (ssa_op->op2_use >= 0) {
6439 if (opline->opcode == ZEND_ASSIGN) {
6440 if (!ra[ssa_op->op2_use]
6441 || ra[ssa_op->op2_use]->reg != ra[ssa_op->op2_def]->reg) {
6442 flags |= ZREG_LOAD;
6443 }
6444 }
6445 }
6446 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def]->reg, flags);
6447 }
6448 }
6449 if (type == IS_LONG
6450 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) {
6451 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6452 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6453 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6454 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6455 ssa->var_info[ssa_op->op2_def].has_range = 1;
6456 }
6457 }
6458
6459 switch (opline->opcode) {
6460 case ZEND_ASSIGN_DIM:
6461 case ZEND_ASSIGN_OBJ:
6462 case ZEND_ASSIGN_STATIC_PROP:
6463 case ZEND_ASSIGN_DIM_OP:
6464 case ZEND_ASSIGN_OBJ_OP:
6465 case ZEND_ASSIGN_STATIC_PROP_OP:
6466 case ZEND_ASSIGN_OBJ_REF:
6467 case ZEND_ASSIGN_STATIC_PROP_REF:
6468 /* OP_DATA */
6469 ssa_op++;
6470 opline++;
6471 if (ssa_op->op1_def >= 0) {
6472 uint8_t type = IS_UNKNOWN;
6473
6474 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6475 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6476 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6477 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6478 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6479 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6480 /* keep old value */
6481 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6482 }
6483 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6484 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->op1_def]));
6485 if (type != IS_UNKNOWN) {
6486 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6487 if (ra && ra[ssa_op->op1_def]) {
6488 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6489 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6490 }
6491 }
6492 if (type == IS_LONG
6493 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6494 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6495 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6496 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6497 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6498 ssa->var_info[ssa_op->op1_def].has_range = 1;
6499 }
6500 }
6501 ssa_op++;
6502 break;
6503 case ZEND_RECV_INIT:
6504 ssa_op++;
6505 opline++;
6506 while (opline->opcode == ZEND_RECV_INIT) {
6507 if (ssa_op->result_def >= 0) {
6508 uint8_t type = IS_UNKNOWN;
6509
6510 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6511 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6512 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6513 }
6514 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6515 (gen_handler || !ra || !ra[ssa_op->result_def]));
6516 if (ra && ra[ssa_op->result_def]) {
6517 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6518 ra[ssa_op->result_def]->flags & ZREG_STORE);
6519 }
6520 }
6521 ssa_op++;
6522 opline++;
6523 }
6524 break;
6525 case ZEND_BIND_GLOBAL:
6526 ssa_op++;
6527 opline++;
6528 while (opline->opcode == ZEND_BIND_GLOBAL) {
6529 if (ssa_op->op1_def >= 0) {
6530 uint8_t type = IS_UNKNOWN;
6531
6532 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6533 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6534 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6535 }
6536 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6537 (gen_handler || !ra || !ra[ssa_op->op1_def]));
6538 if (ra && ra[ssa_op->op1_def]) {
6539 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6540 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6541 }
6542 }
6543 ssa_op++;
6544 opline++;
6545 }
6546 break;
6547 default:
6548 ssa_op += zend_jit_trace_op_len(opline);
6549 break;
6550 }
6551
6552 if (send_result) {
6553 ssa_op++;
6554 p++;
6555 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6556 p++;
6557 }
6558 send_result = 0;
6559 }
6560 }
6561 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6562 call = frame->call;
6563 assert(call && &call->func->op_array == p->op_array);
6564
6565 if (opline->opcode == ZEND_DO_UCALL
6566 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6567 || opline->opcode == ZEND_DO_FCALL) {
6568
6569 frame->call_opline = opline;
6570
6571 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6572 if (opline > op_array->opcodes) {
6573 const zend_op *prev_opline = opline - 1;
6574
6575 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6576 prev_opline--;
6577 }
6578 JIT_G(current_frame) = call;
6579 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6580 || prev_opline->opcode == ZEND_SEND_UNPACK
6581 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6582 && p->op_array->num_args
6583 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6584 && ((p+1)->op == ZEND_JIT_TRACE_VM
6585 || (p+1)->op == ZEND_JIT_TRACE_END)
6586 && (TRACE_FRAME_NUM_ARGS(call) < 0
6587 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6588 && !zend_jit_trace_opline_guard(&dasm_state, (p+1)->opline)) {
6589 goto jit_failure;
6590 }
6591 JIT_G(current_frame) = frame;
6592 }
6593 }
6594
6595 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6596 p++;
6597 break;
6598 }
6599 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6600 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6601 TRACE_FRAME_SET_THIS_CHECKED(call);
6602 }
6603 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6604 TRACE_FRAME_SET_THIS_CHECKED(call);
6605 }
6606 op_array = (zend_op_array*)p->op_array;
6607 jit_extension =
6608 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6609 op_array_ssa = &jit_extension->func_info.ssa;
6610 frame->call = call->prev;
6611 call->prev = frame;
6612 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6613 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6614 } else {
6615 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6616 }
6617 JIT_G(current_frame) = frame = call;
6618 stack = frame->stack;
6619 if (ra) {
6620 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6621
6622 for (i = 0; i < op_array->last_var; i++,j++) {
6623 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6624 if ((ssa->var_info[j].type & MAY_BE_GUARD) != 0) {
6625 uint8_t op_type;
6626
6627 ssa->var_info[j].type &= ~MAY_BE_GUARD;
6628 op_type = concrete_type(ssa->var_info[j].type);
6629 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
6630 goto jit_failure;
6631 }
6632 SET_STACK_TYPE(stack, i, op_type, 1);
6633 }
6634 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6635 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6636 goto jit_failure;
6637 }
6638 }
6639 }
6640 }
6641 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6642 op_array = (zend_op_array*)p->op_array;
6643 jit_extension =
6644 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6645 op_array_ssa = &jit_extension->func_info.ssa;
6646 top = frame;
6647 if (frame->prev) {
6648 checked_stack = frame->old_checked_stack;
6649 peek_checked_stack = frame->old_peek_checked_stack;
6650 frame = frame->prev;
6651 stack = frame->stack;
6652 ZEND_ASSERT(&frame->func->op_array == op_array);
6653 } else {
6654 frame = zend_jit_trace_ret_frame(frame, op_array);
6655 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6656 frame->used_stack = checked_stack = peek_checked_stack = 0;
6657 stack = frame->stack;
6658 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6659 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6660
6661 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6662 /* Initialize abstract stack using SSA */
6663 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6664 && has_concrete_type(ssa->var_info[j].type)) {
6665 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6666 } else {
6667 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6668 }
6669 }
6670 if (ra) {
6671 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6672 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6673 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6674 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6675 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6676 goto jit_failure;
6677 }
6678 }
6679 }
6680 }
6681 } else {
6682 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6683 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6684 }
6685 }
6686 opline = NULL;
6687 }
6688 JIT_G(current_frame) = frame;
6689 if (res_type != IS_UNKNOWN
6690 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6691 const zend_op *opline = (p+1)->opline - 1;
6692 if (opline->result_type != IS_UNUSED) {
6693 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6694 }
6695 }
6696 res_type = IS_UNKNOWN;
6697 } else if (p->op == ZEND_JIT_TRACE_END) {
6698 break;
6699 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6700 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6701 int num_args = -1;
6702
6703 if (init_opline
6704 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6705 num_args = init_opline->extended_value;
6706 }
6707
6708 call = top;
6709 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6710 call->prev = frame->call;
6711 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6712 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6713 if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
6714 TRACE_FRAME_SET_CLOSURE_CALL(call);
6715 }
6716 }
6717 if (init_opline) {
6718 if (init_opline->opcode != ZEND_NEW
6719 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6720 || init_opline->op1_type == IS_UNDEF
6721 || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6722 && ssa_op
6723 && (ssa_op-1)->op1_use >=0
6724 && ssa->var_info[(ssa_op-1)->op1_use].delayed_fetch_this))
6725 && (init_opline->opcode != ZEND_INIT_USER_CALL
6726 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6727 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6728 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6729 ) {
6730 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6731 } else if (init_opline->opcode == ZEND_NEW
6732 || (init_opline->opcode == ZEND_INIT_METHOD_CALL
6733 && init_opline->op1_type != IS_UNDEF
6734 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6735 && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) {
6736 TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call);
6737 }
6738 }
6739 frame->call = call;
6740 top = zend_jit_trace_call_frame(top, p->op_array);
6741 if (p->func) {
6742 if (p->func->type == ZEND_USER_FUNCTION) {
6743 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6744 zend_jit_op_array_trace_extension *jit_extension =
6745 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6746
6747 i = 0;
6748 while (i < p->op_array->num_args) {
6749 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6750 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6751 i++;
6752 }
6753 while (i < p->op_array->last_var) {
6754 if (jit_extension
6755 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6756 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6757 } else {
6758 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6759 }
6760 i++;
6761 }
6762 while (i < p->op_array->last_var + p->op_array->T) {
6763 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6764 i++;
6765 }
6766 } else {
6767 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
6768 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6769 }
6770 }
6771 } else {
6772 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
6773 for (i = 0; i < p->op_array->num_args; i++) {
6774 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6775 }
6776 }
6777 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6778 int skip_guard = 0;
6779
6780 if (init_opline) {
6781 zend_call_info *call_info = jit_extension->func_info.callee_info;
6782
6783 while (call_info) {
6784 if (call_info->caller_init_opline == init_opline
6785 && !call_info->is_prototype) {
6786 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
6787 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
6788 && init_opline->op1_type != IS_CONST) {
6789 break;
6790 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
6791 break;
6792 }
6793 }
6794 skip_guard = 1;
6795 break;
6796 }
6797 call_info = call_info->next_callee;
6798 }
6799 if (!skip_guard
6800 && !zend_jit_may_be_polymorphic_call(init_opline)
6801 && !zend_jit_may_be_modified(p->func, op_array)) {
6802 skip_guard = 1;
6803 }
6804 }
6805
6806 if (!skip_guard) {
6807 if (!opline) {
6808 zend_jit_trace_rec *q = p + 1;
6809 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
6810 q++;
6811 }
6812 opline = q->opline;
6813 ZEND_ASSERT(opline != NULL);
6814 }
6815 if (!zend_jit_init_fcall_guard(&dasm_state,
6816 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
6817 goto jit_failure;
6818 }
6819 }
6820 }
6821 }
6822 call->old_checked_stack = checked_stack;
6823 call->old_peek_checked_stack = peek_checked_stack;
6824 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6825 frame->call_level++;
6826 call->used_stack = checked_stack = peek_checked_stack = 0;
6827 } else {
6828 if (p->func) {
6829 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
6830 } else {
6831 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
6832 }
6833 switch (init_opline->opcode) {
6834 case ZEND_INIT_FCALL:
6835 case ZEND_INIT_FCALL_BY_NAME:
6836 case ZEND_INIT_NS_FCALL_BY_NAME:
6837 case ZEND_INIT_METHOD_CALL:
6838 case ZEND_INIT_DYNAMIC_CALL:
6839 //case ZEND_INIT_STATIC_METHOD_CALL:
6840 //case ZEND_INIT_USER_CALL:
6841 //case ZEND_NEW:
6842 checked_stack += call->used_stack;
6843 if (checked_stack > peek_checked_stack) {
6844 peek_checked_stack = checked_stack;
6845 }
6846 break;
6847 default:
6848 checked_stack = peek_checked_stack = 0;
6849 }
6850 }
6851 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
6852 call = frame->call;
6853 if (call) {
6854 checked_stack = call->old_checked_stack;
6855 peek_checked_stack = call->old_peek_checked_stack;
6856 top = call;
6857 frame->call = call->prev;
6858 }
6859 } else {
6860 ZEND_UNREACHABLE();
6861 }
6862 }
6863
6864 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
6865
6866 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
6867
6868 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6869 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6870 }
6871
6872 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
6873 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
6874 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6875 if (ra) {
6876 zend_ssa_phi *phi = ssa->blocks[1].phis;
6877
6878 while (phi) {
6879 if (ra[phi->ssa_var]
6880 && ra[phi->sources[1]]
6881 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
6882 && (ra[phi->ssa_var]->flags & (ZREG_LOAD|ZREG_STORE)) == 0
6883 && (ra[phi->sources[1]]->flags & (ZREG_LOAD|ZREG_STORE)) == 0) {
6884 /* Store actual type to memory to avoid deoptimization mistakes */
6885 /* TODO: Alternatively, we may try to update alredy generated deoptimization info */
6886 zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var));
6887 }
6888 phi = phi->next;
6889 }
6890 }
6891 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6892 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6893 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6894 goto jit_failure;
6895 }
6896 }
6897 t->link = ZEND_JIT_TRACE_NUM;
6898 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6899 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
6900 }
6901 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
6902 const void *timeout_exit_addr = NULL;
6903
6904 t->flags |= ZEND_JIT_TRACE_LOOP;
6905
6906 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6907 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6908 || (ra
6909 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
6910 /* Deoptimize to the first instruction of the loop */
6911 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
6912
6913 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6914 if (!timeout_exit_addr) {
6915 goto jit_failure;
6916 }
6917 } else {
6918 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6919 }
6920 }
6921
6922 zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
6923 }
6924 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
6925 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
6926 if (ra
6927 && (p-1)->op != ZEND_JIT_TRACE_ENTER
6928 && (p-1)->op != ZEND_JIT_TRACE_BACK
6929 && opline->opcode != ZEND_DO_UCALL
6930 && opline->opcode != ZEND_DO_FCALL
6931 && opline->opcode != ZEND_DO_FCALL_BY_NAME
6932 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
6933 if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL,
6934 stack, op_array->last_var + op_array->T, NULL, NULL, NULL, 0)) {
6935 goto jit_failure;
6936 }
6937 for (i = 0; i < op_array->last_var; i++) {
6938 int8_t reg = STACK_REG(stack, i);
6939 uint8_t type = STACK_TYPE(stack, i);
6940
6941 if (reg == ZREG_NONE
6942 && type != IS_UNKNOWN
6943 && type != STACK_MEM_TYPE(stack, i)) {
6944 if (!zend_jit_store_var_type(&dasm_state, i, type)) {
6945 return 0;
6946 }
6947 SET_STACK_TYPE(stack, i, type, 1);
6948 }
6949 }
6950 }
6951 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
6952 const void *timeout_exit_addr = NULL;
6953
6954 t->link = zend_jit_find_trace(p->opline->handler);
6955 if (t->link == 0) {
6956 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
6957 * by zend_jit_trace_exit() in another thread after this
6958 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
6959 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
6960 * the "_counter_handler" functions, and these are not registered
6961 * tracer functions */
6962 goto jit_failure;
6963 }
6964 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6965 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6966 goto jit_failure;
6967 }
6968 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6969 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6970 }
6971 if (parent_trace
6972 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
6973 && zend_jit_traces[parent_trace].root == t->link) {
6974 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
6975 uint32_t exit_point;
6976
6977 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6978 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6979 }
6980 exit_point = zend_jit_trace_get_exit_point(zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM);
6981 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6982 if (!timeout_exit_addr) {
6983 goto jit_failure;
6984 }
6985 } else {
6986 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6987 }
6988 }
6989 zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
6990 } else {
6991 zend_jit_trace_return(&dasm_state, 0, NULL);
6992 }
6993 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
6994 zend_jit_trace_return(&dasm_state, 0, NULL);
6995 } else {
6996 // TODO: not implemented ???
6997 ZEND_ASSERT(0 && p->stop);
6998 }
6999
7000 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7001 goto jit_failure;
7002 }
7003
7004 if (!zend_jit_trace_end(&dasm_state, t)) {
7005 goto jit_failure;
7006 }
7007
7008 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM,
7009 parent_trace ? SP_ADJ_JIT : ((zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET),
7010 parent_trace ? SP_ADJ_NONE : SP_ADJ_JIT);
7011
7012 if (handler) {
7013 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
7014 const zend_op_array *rec_op_array;
7015
7016 rec_op_array = op_array = trace_buffer->op_array;
7017 jit_extension =
7018 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7019 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7020 for (;;p++) {
7021 if (p->op == ZEND_JIT_TRACE_VM) {
7022 opline = p->opline;
7023 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7024 if (p->op_array == rec_op_array) {
7025 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7026 }
7027 op_array = p->op_array;
7028 jit_extension =
7029 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7030 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7031 op_array = p->op_array;
7032 jit_extension =
7033 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7034 } else if (p->op == ZEND_JIT_TRACE_END) {
7035 break;
7036 }
7037 }
7038 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7039 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7040 if (opline
7041 && (opline->opcode == ZEND_DO_UCALL
7042 || opline->opcode == ZEND_DO_FCALL
7043 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7044 || opline->opcode == ZEND_YIELD
7045 || opline->opcode == ZEND_YIELD_FROM
7046 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7047 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7048 }
7049 if (JIT_G(current_frame)
7050 && JIT_G(current_frame)->prev) {
7051 frame = JIT_G(current_frame)->prev;
7052 do {
7053 if (frame->call_opline) {
7054 op_array = &frame->func->op_array;
7055 jit_extension =
7056 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7057 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7058 }
7059 frame = frame->prev;
7060 } while (frame);
7061 }
7062 }
7063 }
7064
7065 jit_failure:
7066 dasm_free(&dasm_state);
7067
7068 if (name) {
7069 zend_string_release(name);
7070 }
7071
7072 jit_cleanup:
7073 /* Clean up used op_arrays */
7074 while (num_op_arrays > 0) {
7075 op_array = op_arrays[--num_op_arrays];
7076 jit_extension =
7077 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7078
7079 jit_extension->func_info.num = 0;
7080 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7081 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7082 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7083 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7084 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7085 }
7086
7087 zend_arena_release(&CG(arena), checkpoint);
7088
7089 JIT_G(current_frame) = NULL;
7090 JIT_G(current_trace) = NULL;
7091
7092 return handler;
7093 }
7094
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7095 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7096 {
7097 const void *handler = NULL;
7098 dasm_State* dasm_state = NULL;
7099 void *checkpoint;
7100 char name[32];
7101 const zend_op *opline;
7102 uint32_t stack_size;
7103 zend_jit_trace_stack *stack;
7104 bool original_handler = 0;
7105
7106 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7107 return dasm_labels[zend_lbtrace_escape];
7108 }
7109
7110 checkpoint = zend_arena_checkpoint(CG(arena));;
7111
7112 sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num);
7113
7114 dasm_init(&dasm_state, DASM_MAXSECTION);
7115 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
7116 dasm_setup(&dasm_state, dasm_actions);
7117
7118 zend_jit_align_func(&dasm_state);
7119
7120 /* Deoptimization */
7121 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7122 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
7123
7124 if (!zend_jit_trace_deoptimization(&dasm_state,
7125 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7126 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7127 stack, stack_size, NULL, NULL, NULL, 0)) {
7128 goto jit_failure;
7129 }
7130
7131 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7132 if (opline) {
7133 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7134 /* prevent endless loop */
7135 original_handler = 1;
7136 }
7137 zend_jit_set_ip_ex(&dasm_state, opline, original_handler);
7138 }
7139
7140 zend_jit_trace_return(&dasm_state, original_handler, opline);
7141
7142 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM, SP_ADJ_JIT, SP_ADJ_NONE);
7143
7144 jit_failure:
7145 dasm_free(&dasm_state);
7146 zend_arena_release(&CG(arena), checkpoint);
7147 return handler;
7148 }
7149
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7150 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7151 {
7152 zend_jit_trace_stop ret;
7153 const void *handler;
7154 uint8_t orig_trigger;
7155 zend_jit_trace_info *t = NULL;
7156 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7157 bool do_bailout = 0;
7158
7159 zend_shared_alloc_lock();
7160
7161 /* Checks under lock */
7162 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7163 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7164 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7165 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7166 } else {
7167 zend_try {
7168 SHM_UNPROTECT();
7169 zend_jit_unprotect();
7170
7171 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7172
7173 t->id = ZEND_JIT_TRACE_NUM;
7174 t->root = ZEND_JIT_TRACE_NUM;
7175 t->parent = 0;
7176 t->link = 0;
7177 t->exit_count = 0;
7178 t->child_count = 0;
7179 t->stack_map_size = 0;
7180 t->flags = 0;
7181 t->polymorphism = 0;
7182 t->jmp_table_size = 0;
7183 t->op_array = trace_buffer[0].op_array;
7184 t->opline = trace_buffer[1].opline;
7185 t->exit_info = exit_info;
7186 t->stack_map = NULL;
7187
7188 orig_trigger = JIT_G(trigger);
7189 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7190
7191 handler = zend_jit_trace(trace_buffer, 0, 0);
7192
7193 JIT_G(trigger) = orig_trigger;
7194
7195 if (handler) {
7196 zend_jit_trace_exit_info *shared_exit_info = NULL;
7197
7198 t->exit_info = NULL;
7199 if (t->exit_count) {
7200 /* reallocate exit_info into shared memory */
7201 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7202 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7203
7204 if (!shared_exit_info) {
7205 if (t->stack_map) {
7206 efree(t->stack_map);
7207 t->stack_map = NULL;
7208 }
7209 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7210 goto exit;
7211 }
7212 memcpy(shared_exit_info, exit_info,
7213 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7214 t->exit_info = shared_exit_info;
7215 }
7216
7217 if (t->stack_map_size) {
7218 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7219 if (!shared_stack_map) {
7220 efree(t->stack_map);
7221 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7222 goto exit;
7223 }
7224 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7225 efree(t->stack_map);
7226 t->stack_map = shared_stack_map;
7227 }
7228
7229 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7230 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7231
7232 ((zend_op*)opline)->handler = handler;
7233
7234 ZEND_JIT_TRACE_NUM++;
7235 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7236
7237 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7238 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7239 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7240 if (t->stack_map) {
7241 efree(t->stack_map);
7242 t->stack_map = NULL;
7243 }
7244 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7245 } else {
7246 if (t->stack_map) {
7247 efree(t->stack_map);
7248 t->stack_map = NULL;
7249 }
7250 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7251 }
7252
7253 exit:;
7254 } zend_catch {
7255 do_bailout = 1;
7256 } zend_end_try();
7257
7258 zend_jit_protect();
7259 SHM_PROTECT();
7260 }
7261
7262 zend_shared_alloc_unlock();
7263
7264 if (do_bailout) {
7265 zend_bailout();
7266 }
7267
7268 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7269 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7270 && t->exit_count > 0) {
7271 zend_jit_dump_exit_info(t);
7272 }
7273
7274 return ret;
7275 }
7276
7277 /* Set counting handler back to original VM handler. */
zend_jit_stop_hot_trace_counters(zend_op_array * op_array)7278 static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array)
7279 {
7280 zend_jit_op_array_trace_extension *jit_extension;
7281 uint32_t i;
7282
7283 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7284 for (i = 0; i < op_array->last; i++) {
7285 /* Opline with Jit-ed code handler is skipped. */
7286 if (jit_extension->trace_info[i].trace_flags &
7287 (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7288 continue;
7289 }
7290 if (jit_extension->trace_info[i].trace_flags &
7291 (ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_START_RETURN)) {
7292 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7293 }
7294 }
7295 }
7296
7297 /* Get the tracing op_array. */
zend_jit_stop_persistent_op_array(zend_op_array * op_array)7298 static void zend_jit_stop_persistent_op_array(zend_op_array *op_array)
7299 {
7300 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
7301 if (!func_info) {
7302 return;
7303 }
7304 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
7305 zend_jit_stop_hot_trace_counters(op_array);
7306 }
7307 }
7308
7309 /* Get all op_arrays with counter handler. */
zend_jit_stop_persistent_script(zend_persistent_script * script)7310 static void zend_jit_stop_persistent_script(zend_persistent_script *script)
7311 {
7312 zend_class_entry *ce;
7313 zend_op_array *op_array;
7314
7315 zend_jit_stop_persistent_op_array(&script->script.main_op_array);
7316
7317 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
7318 zend_jit_stop_persistent_op_array(op_array);
7319 } ZEND_HASH_FOREACH_END();
7320
7321 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
7322 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
7323 if (op_array->type == ZEND_USER_FUNCTION) {
7324 zend_jit_stop_persistent_op_array(op_array);
7325 }
7326 } ZEND_HASH_FOREACH_END();
7327 } ZEND_HASH_FOREACH_END();
7328 }
7329
7330 /* Get all scripts which are accelerated by JIT */
zend_jit_stop_counter_handlers(void)7331 static void zend_jit_stop_counter_handlers(void)
7332 {
7333 if (ZCSG(jit_counters_stopped)) {
7334 return;
7335 }
7336
7337 zend_shared_alloc_lock();
7338 /* mprotect has an extreme overhead, avoid calls to it for every function. */
7339 SHM_UNPROTECT();
7340 if (!ZCSG(jit_counters_stopped)) {
7341 ZCSG(jit_counters_stopped) = true;
7342 for (uint32_t i = 0; i < ZCSG(hash).max_num_entries; i++) {
7343 zend_accel_hash_entry *cache_entry;
7344 for (cache_entry = ZCSG(hash).hash_table[i]; cache_entry; cache_entry = cache_entry->next) {
7345 zend_persistent_script *script;
7346 if (cache_entry->indirect) continue;
7347 script = (zend_persistent_script *)cache_entry->data;
7348 zend_jit_stop_persistent_script(script);
7349 }
7350 }
7351 }
7352 SHM_PROTECT();
7353 zend_shared_alloc_unlock();
7354 }
7355
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7356 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7357 {
7358 zend_shared_alloc_lock();
7359
7360 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7361 SHM_UNPROTECT();
7362 zend_jit_unprotect();
7363
7364 ((zend_op*)opline)->handler =
7365 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7366
7367 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7368
7369 zend_jit_protect();
7370 SHM_PROTECT();
7371 }
7372
7373 zend_shared_alloc_unlock();
7374 }
7375
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7376 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7377 {
7378 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7379 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7380 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7381 uint32_t cache_slot = JIT_G(bad_root_slot);
7382 uint32_t i;
7383
7384 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7385 if (cache_opline[i] == opline) {
7386 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7387 cache_opline[i] = NULL;
7388 return 1;
7389 } else {
7390 #if 0
7391 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7392 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7393 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7394 }
7395 #endif
7396 cache_count[i]++;
7397 cache_stop[i] = stop;
7398 return 0;
7399 }
7400 }
7401 }
7402 i = cache_slot;
7403 cache_opline[i] = opline;
7404 cache_count[i] = 1;
7405 cache_stop[i] = stop;
7406 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7407 JIT_G(bad_root_slot) = cache_slot;
7408 return 0;
7409 }
7410
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7411 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7412 {
7413 zend_jit_trace_rec *p = trace_buffer;
7414 const zend_op_array *op_array;
7415 const zend_op *opline;
7416 uint32_t level = 1 + trace_buffer[0].level;
7417 int idx, len, i, v, vars_count, call_level;
7418
7419 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7420 op_array = p->op_array;
7421 p += ZEND_JIT_TRACE_START_REC_SIZE;
7422 idx = 0;
7423 call_level = 0;
7424
7425 if (tssa && tssa->var_info) {
7426 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7427 vars_count = op_array->last_var;
7428 } else {
7429 vars_count = op_array->last_var + op_array->T;
7430 }
7431 for (i = 0; i < vars_count; i++) {
7432 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7433 fprintf(stderr, " %*c;", level, ' ');
7434 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7435 fprintf(stderr, "\n");
7436 }
7437 }
7438 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7439 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7440 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7441 zend_ssa_phi *p = tssa->blocks[1].phis;
7442
7443 fprintf(stderr, "LOOP:\n");
7444
7445 while (p) {
7446 fprintf(stderr, " ;");
7447 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7448 fprintf(stderr, " = Phi(");
7449 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7450 fprintf(stderr, ", ");
7451 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7452 fprintf(stderr, ")\n");
7453 p = p->next;
7454 }
7455 }
7456 }
7457
7458 while (1) {
7459 if (p->op == ZEND_JIT_TRACE_VM) {
7460 uint8_t op1_type, op2_type, op3_type;
7461
7462 opline = p->opline;
7463 fprintf(stderr, "%04d%*c",
7464 (int)(opline - op_array->opcodes),
7465 level, ' ');
7466 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7467
7468 op1_type = p->op1_type;
7469 op2_type = p->op2_type;
7470 op3_type = p->op3_type;
7471 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7472 fprintf(stderr, " ;");
7473 if (op1_type != IS_UNKNOWN) {
7474 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7475 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7476 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7477 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7478 p++;
7479 fprintf(stderr, " op1(%sobject of class %s)", ref,
7480 ZSTR_VAL(p->ce->name));
7481 } else {
7482 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7483 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7484 }
7485 }
7486 if (op2_type != IS_UNKNOWN) {
7487 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7488 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7489 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7490 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7491 p++;
7492 fprintf(stderr, " op2(%sobject of class %s)", ref,
7493 ZSTR_VAL(p->ce->name));
7494 } else {
7495 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7496 fprintf(stderr, " op2(%s%s)", ref, type);
7497 }
7498 }
7499 if (op3_type != IS_UNKNOWN) {
7500 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7501 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7502 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7503 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7504 fprintf(stderr, " op3(%s%s)", ref, type);
7505 }
7506 }
7507 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7508 uint8_t val_type;
7509 const char *type;
7510
7511 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7512 fprintf(stderr, " ;");
7513 }
7514 p++;
7515 val_type = p->op1_type;
7516
7517 if (val_type == IS_UNDEF) {
7518 type = "undef";
7519 } else if (val_type == IS_REFERENCE) {
7520 type = "ref";
7521 } else {
7522 type = zend_get_type_by_const(val_type);
7523 }
7524 fprintf(stderr, " val(%s)", type);
7525 }
7526 fprintf(stderr, "\n");
7527 idx++;
7528
7529 len = zend_jit_trace_op_len(opline);
7530 while (len > 1) {
7531 opline++;
7532 fprintf(stderr, "%04d%*c;",
7533 (int)(opline - op_array->opcodes),
7534 level, ' ');
7535 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7536 idx++;
7537 len--;
7538 fprintf(stderr, "\n");
7539 }
7540 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7541 op_array = p->op_array;
7542 fprintf(stderr, " %*c>enter %s%s%s\n",
7543 level, ' ',
7544 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7545 op_array->scope ? "::" : "",
7546 op_array->function_name ?
7547 ZSTR_VAL(op_array->function_name) :
7548 ZSTR_VAL(op_array->filename));
7549 level++;
7550 if (tssa && tssa->var_info) {
7551 call_level++;
7552 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7553 vars_count = op_array->last_var;
7554 for (i = 0; i < vars_count; i++, v++) {
7555 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7556 fprintf(stderr, " %*c;", level, ' ');
7557 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7558 fprintf(stderr, "\n");
7559 }
7560 }
7561 }
7562 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7563 op_array = p->op_array;
7564 level--;
7565 fprintf(stderr, " %*c<back %s%s%s\n",
7566 level, ' ',
7567 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7568 op_array->scope ? "::" : "",
7569 op_array->function_name ?
7570 ZSTR_VAL(op_array->function_name) :
7571 ZSTR_VAL(op_array->filename));
7572 if (tssa && tssa->var_info) {
7573 if (call_level == 0) {
7574 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7575 vars_count = op_array->last_var + op_array->T;
7576 for (i = 0; i < vars_count; i++, v++) {
7577 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7578 fprintf(stderr, " %*c;", level, ' ');
7579 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7580 fprintf(stderr, "\n");
7581 }
7582 }
7583 } else {
7584 call_level--;
7585 }
7586 }
7587 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7588 if (p->func != (zend_function*)&zend_pass_function) {
7589 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7590 level, ' ',
7591 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7592 (p->func && p->func->common.scope) ? "::" : "",
7593 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7594 } else {
7595 fprintf(stderr, " %*c>skip\n",
7596 level, ' ');
7597 }
7598 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7599 if (p->func != (zend_function*)&zend_pass_function) {
7600 fprintf(stderr, " %*c>call %s%s%s\n",
7601 level, ' ',
7602 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7603 p->func->common.scope ? "::" : "",
7604 ZSTR_VAL(p->func->common.function_name));
7605 } else {
7606 fprintf(stderr, " %*c>skip\n",
7607 level, ' ');
7608 }
7609 } else if (p->op == ZEND_JIT_TRACE_END) {
7610 break;
7611 }
7612 p++;
7613 }
7614 }
7615
zend_jit_dump_exit_info(zend_jit_trace_info * t)7616 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7617 {
7618 int i, j;
7619
7620 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7621 for (i = 0; i < t->exit_count; i++) {
7622 const zend_op_array *op_array = t->exit_info[i].op_array;
7623 uint32_t stack_size = t->exit_info[i].stack_size;
7624 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset;
7625
7626 fprintf(stderr, " exit_%d:", i);
7627 if (t->exit_info[i].opline) {
7628 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7629 } else {
7630 fprintf(stderr, " ----/");
7631 }
7632 if (t->exit_info[i].stack_size) {
7633 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7634 } else {
7635 fprintf(stderr, "----/0");
7636 }
7637 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7638 fprintf(stderr, "/VM");
7639 }
7640 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7641 fprintf(stderr, "/CALL");
7642 }
7643 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7644 fprintf(stderr, "/POLY");
7645 }
7646 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7647 fprintf(stderr, "/FREE_OP1");
7648 }
7649 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7650 fprintf(stderr, "/FREE_OP2");
7651 }
7652 for (j = 0; j < stack_size; j++) {
7653 uint8_t type = STACK_TYPE(stack, j);
7654 if (type != IS_UNKNOWN) {
7655 fprintf(stderr, " ");
7656 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7657 fprintf(stderr, ":");
7658 if (type == IS_UNDEF) {
7659 fprintf(stderr, "undef");
7660 } else {
7661 fprintf(stderr, "%s", zend_get_type_by_const(type));
7662 }
7663 if (STACK_REG(stack, j) != ZREG_NONE) {
7664 if (STACK_REG(stack, j) < ZREG_NUM) {
7665 fprintf(stderr, "(%s)", zend_reg_name[STACK_REG(stack, j)]);
7666 } else if (STACK_REG(stack, j) == ZREG_THIS) {
7667 fprintf(stderr, "(this)");
7668 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7669 fprintf(stderr, "(zval_try_addref)");
7670 } else {
7671 fprintf(stderr, "(const_%d)", STACK_REG(stack, j) - ZREG_NUM);
7672 }
7673 }
7674 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7675 fprintf(stderr, " ");
7676 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7677 fprintf(stderr, ":unknown(zval_try_addref)");
7678 } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_GPR0) {
7679 fprintf(stderr, " ");
7680 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7681 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[ZREG_COPY]);
7682 }
7683 }
7684 fprintf(stderr, "\n");
7685 }
7686 }
7687
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)7688 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
7689 {
7690 const zend_op *orig_opline;
7691 zend_jit_trace_stop stop;
7692 int ret = 0;
7693 zend_op_array *op_array;
7694 zend_jit_op_array_trace_extension *jit_extension;
7695 size_t offset;
7696 uint32_t trace_num;
7697 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7698
7699 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
7700 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
7701 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
7702
7703 repeat:
7704 trace_num = ZEND_JIT_TRACE_NUM;
7705 orig_opline = opline;
7706 op_array = &EX(func)->op_array;
7707 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7708 offset = jit_extension->offset;
7709
7710 EX(opline) = opline;
7711
7712 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
7713 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7714 return 0;
7715 }
7716
7717 if (JIT_G(tracing)) {
7718 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
7719 return 0;
7720 }
7721
7722 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7723 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7724 trace_num,
7725 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7726 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7727 EX(func)->op_array.scope ? "::" : "",
7728 EX(func)->op_array.function_name ?
7729 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7730 ZSTR_VAL(EX(func)->op_array.filename),
7731 opline->lineno);
7732 }
7733
7734 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7735 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7736 zend_jit_stop_counter_handlers();
7737 goto abort;
7738 }
7739
7740 JIT_G(tracing) = 1;
7741 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
7742 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
7743 JIT_G(tracing) = 0;
7744
7745 if (stop & ZEND_JIT_TRACE_HALT) {
7746 ret = -1;
7747 }
7748 stop &= ~ZEND_JIT_TRACE_HALT;
7749
7750 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
7751 orig_opline = trace_buffer[1].opline;
7752 op_array = (zend_op_array*)trace_buffer[0].op_array;
7753 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7754 offset = jit_extension->offset;
7755 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7756 const zend_op_array *op_array = trace_buffer[0].op_array;
7757 const zend_op *opline = trace_buffer[1].opline;
7758 zend_jit_op_array_trace_extension *jit_extension =
7759 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7760 size_t offset = jit_extension->offset;
7761
7762 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7763 trace_num,
7764 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7765 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7766 op_array->scope ? "::" : "",
7767 op_array->function_name ?
7768 ZSTR_VAL(op_array->function_name) : "$main",
7769 ZSTR_VAL(op_array->filename),
7770 opline->lineno);
7771 }
7772 }
7773
7774 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
7775 zend_jit_dump_trace(trace_buffer, NULL);
7776 }
7777
7778 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
7779 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
7780 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
7781 uint32_t idx = trace_buffer[1].last;
7782 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
7783 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
7784 trace_num,
7785 link_to);
7786 } else {
7787 fprintf(stderr, "---- TRACE %d stop (%s)\n",
7788 trace_num,
7789 zend_jit_trace_stop_description[stop]);
7790 }
7791 }
7792 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
7793 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
7794 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
7795 fprintf(stderr, "---- TRACE %d %s\n",
7796 trace_num,
7797 zend_jit_trace_stop_description[stop]);
7798 }
7799 } else {
7800 goto abort;
7801 }
7802 } else {
7803 abort:
7804 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
7805 fprintf(stderr, "---- TRACE %d abort (%s)\n",
7806 trace_num,
7807 zend_jit_trace_stop_description[stop]);
7808 }
7809 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
7810 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
7811 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7812 fprintf(stderr, "---- TRACE %d blacklisted\n",
7813 trace_num);
7814 }
7815 zend_jit_blacklist_root_trace(orig_opline, offset);
7816 }
7817 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
7818 execute_data = EG(current_execute_data);
7819 opline = EX(opline);
7820 goto repeat;
7821 }
7822 }
7823
7824 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
7825 fprintf(stderr, "\n");
7826 }
7827
7828 return ret;
7829 }
7830
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)7831 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
7832 {
7833 const void *handler;
7834 bool do_bailout = 0;
7835
7836 zend_shared_alloc_lock();
7837
7838 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
7839 SHM_UNPROTECT();
7840 zend_jit_unprotect();
7841
7842 zend_try {
7843 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
7844
7845 if (handler) {
7846 zend_jit_link_side_trace(
7847 zend_jit_traces[trace_num].code_start,
7848 zend_jit_traces[trace_num].code_size,
7849 zend_jit_traces[trace_num].jmp_table_size,
7850 exit_num,
7851 handler);
7852 }
7853 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
7854 } zend_catch {
7855 do_bailout = 1;
7856 } zend_end_try();
7857
7858 zend_jit_protect();
7859 SHM_PROTECT();
7860 }
7861
7862 zend_shared_alloc_unlock();
7863
7864 if (do_bailout) {
7865 zend_bailout();
7866 }
7867 }
7868
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)7869 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
7870 {
7871 uint8_t *counter = JIT_G(exit_counters) +
7872 zend_jit_traces[trace_num].exit_counters + exit_num;
7873
7874 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
7875 return 1;
7876 }
7877 (*counter)++;
7878 return 0;
7879 }
7880
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)7881 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
7882 {
7883 uint8_t *counter = JIT_G(exit_counters) +
7884 zend_jit_traces[trace_num].exit_counters + exit_num;
7885
7886 if (*counter + 1 >= JIT_G(hot_side_exit)) {
7887 return 1;
7888 }
7889 (*counter)++;
7890 return 0;
7891 }
7892
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)7893 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
7894 {
7895 zend_jit_trace_stop ret;
7896 const void *handler;
7897 uint8_t orig_trigger;
7898 zend_jit_trace_info *t;
7899 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7900 bool do_bailout = 0;
7901
7902 zend_shared_alloc_lock();
7903
7904 /* Checks under lock */
7905 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7906 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7907 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7908 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7909 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
7910 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
7911 } else {
7912 SHM_UNPROTECT();
7913 zend_jit_unprotect();
7914
7915 zend_try {
7916 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7917
7918 t->id = ZEND_JIT_TRACE_NUM;
7919 t->root = zend_jit_traces[parent_num].root;
7920 t->parent = parent_num;
7921 t->link = 0;
7922 t->exit_count = 0;
7923 t->child_count = 0;
7924 t->stack_map_size = 0;
7925 t->flags = 0;
7926 t->polymorphism = polymorphism;
7927 t->jmp_table_size = 0;
7928 t->opline = NULL;
7929 t->exit_info = exit_info;
7930 t->stack_map = NULL;
7931
7932 orig_trigger = JIT_G(trigger);
7933 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7934
7935 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
7936
7937 JIT_G(trigger) = orig_trigger;
7938
7939 if (handler) {
7940 zend_jit_trace_exit_info *shared_exit_info = NULL;
7941
7942 t->exit_info = NULL;
7943 if (t->exit_count) {
7944 /* reallocate exit_info into shared memory */
7945 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7946 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7947
7948 if (!shared_exit_info) {
7949 if (t->stack_map) {
7950 efree(t->stack_map);
7951 t->stack_map = NULL;
7952 }
7953 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7954 goto exit;
7955 }
7956 memcpy(shared_exit_info, exit_info,
7957 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7958 t->exit_info = shared_exit_info;
7959 }
7960
7961 if (t->stack_map_size) {
7962 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7963 if (!shared_stack_map) {
7964 efree(t->stack_map);
7965 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7966 goto exit;
7967 }
7968 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7969 efree(t->stack_map);
7970 t->stack_map = shared_stack_map;
7971 }
7972
7973 zend_jit_link_side_trace(
7974 zend_jit_traces[parent_num].code_start,
7975 zend_jit_traces[parent_num].code_size,
7976 zend_jit_traces[parent_num].jmp_table_size,
7977 exit_num,
7978 handler);
7979
7980 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7981 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7982
7983 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
7984 ZEND_JIT_TRACE_NUM++;
7985 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
7986
7987 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7988 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7989 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7990 if (t->stack_map) {
7991 efree(t->stack_map);
7992 t->stack_map = NULL;
7993 }
7994 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7995 } else {
7996 if (t->stack_map) {
7997 efree(t->stack_map);
7998 t->stack_map = NULL;
7999 }
8000 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
8001 }
8002
8003 exit:;
8004 } zend_catch {
8005 do_bailout = 1;
8006 } zend_end_try();
8007
8008 zend_jit_protect();
8009 SHM_PROTECT();
8010 }
8011
8012 zend_shared_alloc_unlock();
8013
8014 if (do_bailout) {
8015 zend_bailout();
8016 }
8017
8018 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
8019 && ret == ZEND_JIT_TRACE_STOP_COMPILED
8020 && t->exit_count > 0) {
8021 zend_jit_dump_exit_info(t);
8022 }
8023
8024 return ret;
8025 }
8026
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)8027 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
8028 {
8029 zend_jit_trace_stop stop;
8030 int ret = 0;
8031 uint32_t trace_num;
8032 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8033 uint32_t is_megamorphic = 0;
8034 uint32_t polymorphism = 0;
8035
8036 trace_num = ZEND_JIT_TRACE_NUM;
8037
8038 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8039 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8040 return 0;
8041 }
8042
8043 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8044 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
8045 trace_num, parent_num, exit_num,
8046 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8047 EX(func)->op_array.scope ? "::" : "",
8048 EX(func)->op_array.function_name ?
8049 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8050 ZSTR_VAL(EX(func)->op_array.filename),
8051 EX(opline)->lineno);
8052 }
8053
8054 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8055 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8056 goto abort;
8057 }
8058
8059 if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8060 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8061 goto abort;
8062 }
8063
8064 if (JIT_G(max_polymorphic_calls) > 0) {
8065 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
8066 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
8067 && EX(call))) {
8068 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
8069 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
8070 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
8071 } else if (!zend_jit_traces[parent_num].polymorphism) {
8072 polymorphism = 1;
8073 } else if (exit_num == 0) {
8074 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
8075 }
8076 }
8077 }
8078
8079 JIT_G(tracing) = 1;
8080 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
8081 JIT_G(tracing) = 0;
8082
8083 if (stop & ZEND_JIT_TRACE_HALT) {
8084 ret = -1;
8085 }
8086 stop &= ~ZEND_JIT_TRACE_HALT;
8087
8088 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8089 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8090 const zend_op_array *op_array = trace_buffer[0].op_array;
8091 const zend_op *opline = trace_buffer[1].opline;
8092 zend_jit_op_array_trace_extension *jit_extension =
8093 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8094 size_t offset = jit_extension->offset;
8095
8096 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8097 trace_num,
8098 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8099 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8100 op_array->scope ? "::" : "",
8101 op_array->function_name ?
8102 ZSTR_VAL(op_array->function_name) : "$main",
8103 ZSTR_VAL(op_array->filename),
8104 opline->lineno);
8105 }
8106 }
8107
8108 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8109 zend_jit_dump_trace(trace_buffer, NULL);
8110 }
8111
8112 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8113 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8114 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8115 uint32_t idx = trace_buffer[1].last;
8116 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8117 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8118 trace_num,
8119 link_to);
8120 } else {
8121 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8122 trace_num,
8123 zend_jit_trace_stop_description[stop]);
8124 }
8125 }
8126 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8127 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8128 } else {
8129 const zend_op_array *op_array = trace_buffer[0].op_array;
8130 zend_jit_op_array_trace_extension *jit_extension =
8131 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8132 const zend_op *opline = trace_buffer[1].opline;
8133
8134 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8135 }
8136 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8137 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8138 fprintf(stderr, "---- TRACE %d %s\n",
8139 trace_num,
8140 zend_jit_trace_stop_description[stop]);
8141 }
8142 } else {
8143 goto abort;
8144 }
8145 } else {
8146 abort:
8147 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8148 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8149 trace_num,
8150 zend_jit_trace_stop_description[stop]);
8151 }
8152 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8153 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8154 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8155 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8156 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8157 parent_num, exit_num);
8158 }
8159 }
8160 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8161 execute_data = EG(current_execute_data);
8162 return zend_jit_trace_hot_root(execute_data, EX(opline));
8163 }
8164 }
8165
8166 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8167 fprintf(stderr, "\n");
8168 }
8169
8170 return ret;
8171 }
8172
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8173 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8174 {
8175 uint32_t trace_num = EG(jit_trace_num);
8176 zend_execute_data *execute_data = EG(current_execute_data);
8177 const zend_op *orig_opline = EX(opline);
8178 const zend_op *opline;
8179 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8180 int repeat_last_opline = 0;
8181
8182 /* Deoptimization of VM stack state */
8183 uint32_t i;
8184 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8185 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset;
8186
8187 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8188 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8189 call->prev_execute_data = EX(call);
8190 EX(call) = call;
8191 }
8192
8193 for (i = 0; i < stack_size; i++) {
8194 if (STACK_REG(stack, i) != ZREG_NONE) {
8195 if (STACK_TYPE(stack, i) == IS_LONG) {
8196 zend_long val;
8197
8198 if (STACK_REG(stack, i) < ZREG_NUM) {
8199 val = regs->gpr[STACK_REG(stack, i)];
8200 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN) {
8201 val = ZEND_LONG_MIN;
8202 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX) {
8203 val = ZEND_LONG_MAX;
8204 } else {
8205 ZEND_UNREACHABLE();
8206 }
8207 ZVAL_LONG(EX_VAR_NUM(i), val);
8208 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8209 double val;
8210
8211 if (STACK_REG(stack, i) < ZREG_NUM) {
8212 val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8213 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN_MINUS_1) {
8214 val = (double)ZEND_LONG_MIN - 1.0;
8215 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX_PLUS_1) {
8216 val = (double)ZEND_LONG_MAX + 1.0;
8217 } else {
8218 ZEND_UNREACHABLE();
8219 }
8220 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8221 } else if (STACK_REG(stack, i) == ZREG_THIS) {
8222 zend_object *obj = Z_OBJ(EX(This));
8223
8224 GC_ADDREF(obj);
8225 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8226 } else if (STACK_REG(stack, i) == ZREG_NULL) {
8227 ZVAL_NULL(EX_VAR_NUM(i));
8228 } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) {
8229 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8230 } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_GPR0) {
8231 zval *val = (zval*)regs->gpr[ZREG_COPY];
8232
8233 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8234 /* Undefined array index or property */
8235 repeat_last_opline = 1;
8236 } else {
8237 ZVAL_COPY(EX_VAR_NUM(i), val);
8238 }
8239 } else {
8240 ZEND_UNREACHABLE();
8241 }
8242 }
8243 }
8244
8245 if (repeat_last_opline) {
8246 EX(opline) = t->exit_info[exit_num].opline - 1;
8247 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8248 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8249 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8250 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8251 }
8252 return 1;
8253 }
8254
8255 opline = t->exit_info[exit_num].opline;
8256
8257 if (opline) {
8258 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8259 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8260 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8261 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8262 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8263 EX(opline) = opline-1;
8264 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8265 }
8266 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8267 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8268 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8269 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8270 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8271 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8272 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8273 EX(opline) = opline-1;
8274 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8275 }
8276 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8277 if (EG(exception)) {
8278 return 1;
8279 }
8280 }
8281 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8282 zend_function *func = (zend_function*)regs->gpr[ZREG_COPY];
8283
8284 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8285 zend_string_release_ex(func->common.function_name, 0);
8286 zend_free_trampoline(func);
8287 EX(opline) = opline;
8288 return 1;
8289 }
8290 }
8291
8292 /* Set VM opline to continue interpretation */
8293 EX(opline) = opline;
8294 }
8295
8296 if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) {
8297 return 1;
8298 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8299 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8300 return 0;
8301 }
8302
8303 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8304 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8305 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8306
8307 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8308 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8309 trace_num,
8310 exit_num,
8311 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8312 EX(func)->op_array.scope ? "::" : "",
8313 EX(func)->op_array.function_name ?
8314 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8315 ZSTR_VAL(EX(func)->op_array.filename),
8316 EX(opline)->lineno);
8317 }
8318
8319 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8320 zend_jit_op_array_trace_extension *jit_extension;
8321 uint32_t num = trace_num;
8322
8323 while (t->root != num) {
8324 num = t->root;
8325 t = &zend_jit_traces[num];
8326 }
8327
8328 zend_shared_alloc_lock();
8329
8330 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8331
8332 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8333 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8334 /* skip: not JIT-ed nor blacklisted */
8335 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8336 /* too many root traces, blacklist the root trace */
8337 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
8338 SHM_UNPROTECT();
8339 zend_jit_unprotect();
8340
8341 ((zend_op*)opline)->handler =
8342 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->orig_handler;
8343
8344 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= ~ZEND_JIT_TRACE_JITED;
8345 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
8346
8347 zend_jit_protect();
8348 SHM_PROTECT();
8349 }
8350 } else {
8351 SHM_UNPROTECT();
8352 zend_jit_unprotect();
8353
8354 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8355 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8356 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8357 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8358 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8359 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8360 }
8361 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8362 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8363
8364 zend_jit_protect();
8365 SHM_PROTECT();
8366 }
8367
8368 zend_shared_alloc_unlock();
8369
8370 return 0;
8371 }
8372
8373 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8374 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8375 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8376 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8377 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8378 trace_num, exit_num);
8379 }
8380 return 0;
8381 }
8382 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8383 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8384 }
8385
8386 /* Return 1 to call original handler instead of the same JIT-ed trace */
8387 return (orig_opline == t->opline && EX(opline) == orig_opline);
8388 }
8389
zend_jit_trace_supported(const zend_op * opline)8390 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8391 {
8392 switch (opline->opcode) {
8393 case ZEND_CATCH:
8394 case ZEND_FAST_CALL:
8395 case ZEND_FAST_RET:
8396 return ZEND_JIT_TRACE_UNSUPPORTED;
8397 default:
8398 return ZEND_JIT_TRACE_SUPPORTED;
8399 }
8400 }
8401
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8402 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8403 {
8404 zend_jit_op_array_trace_extension *jit_extension;
8405 uint32_t i;
8406
8407 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8408 for (i = 0; i < op_array->last; i++) {
8409 jit_extension->trace_info[i].trace_flags &=
8410 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8411 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8412 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8413 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8414 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8415 } else {
8416 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8417 }
8418 }
8419 return SUCCESS;
8420 }
8421
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8422 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8423 {
8424 zend_op *opline;
8425 zend_jit_op_array_trace_extension *jit_extension;
8426 uint32_t i;
8427
8428 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8429
8430 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8431 if (!jit_extension) {
8432 return FAILURE;
8433 }
8434 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8435 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8436 jit_extension->op_array = op_array;
8437 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8438 for (i = 0; i < op_array->last; i++) {
8439 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8440 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8441 jit_extension->trace_info[i].counter = NULL;
8442 jit_extension->trace_info[i].trace_flags =
8443 zend_jit_trace_supported(&op_array->opcodes[i]);
8444 }
8445 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8446
8447 if (JIT_G(hot_loop)) {
8448 zend_cfg cfg;
8449
8450 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8451
8452 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8453 return FAILURE;
8454 }
8455
8456 for (i = 0; i < cfg.blocks_count; i++) {
8457 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8458 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8459 /* loop header */
8460 opline = op_array->opcodes + cfg.blocks[i].start;
8461 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8462 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8463 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8464 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8465 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8466 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8467 }
8468 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8469 ZEND_JIT_TRACE_START_LOOP;
8470 }
8471 }
8472 }
8473 }
8474 }
8475
8476 if (JIT_G(hot_func)) {
8477 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8478 opline = op_array->opcodes;
8479 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8480 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8481 opline++;
8482 }
8483 }
8484
8485 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8486 /* function entry */
8487 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8488 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8489 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8490 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8491 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8492 ZEND_JIT_TRACE_START_ENTER;
8493 }
8494 }
8495
8496 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8497
8498 return SUCCESS;
8499 }
8500
zend_jit_trace_init_caches(void)8501 static void zend_jit_trace_init_caches(void)
8502 {
8503 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8504 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8505 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8506 JIT_G(bad_root_slot) = 0;
8507
8508 if (JIT_G(exit_counters)) {
8509 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8510 }
8511 }
8512
zend_jit_trace_reset_caches(void)8513 static void zend_jit_trace_reset_caches(void)
8514 {
8515 JIT_G(tracing) = 0;
8516 #ifdef ZTS
8517 if (!JIT_G(exit_counters)) {
8518 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8519 }
8520 #endif
8521 }
8522
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8523 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8524 {
8525 if (jit_globals->exit_counters) {
8526 free(jit_globals->exit_counters);
8527 }
8528 }
8529
zend_jit_trace_restart(void)8530 static void zend_jit_trace_restart(void)
8531 {
8532 ZEND_JIT_TRACE_NUM = 1;
8533 ZEND_JIT_COUNTER_NUM = 0;
8534 ZEND_JIT_EXIT_NUM = 0;
8535 ZEND_JIT_EXIT_COUNTERS = 0;
8536 ZCSG(jit_counters_stopped) = false;
8537
8538 zend_jit_trace_init_caches();
8539 }
8540
