1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(bool reattached)49 static int zend_jit_trace_startup(bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 return FAILURE;
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 return FAILURE;
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 ZCSG(jit_counters_stopped) = false;
67 } else {
68 zend_jit_traces = ZCSG(jit_traces);
69 if (!zend_jit_traces) {
70 return FAILURE;
71 }
72 zend_jit_exit_groups = ZCSG(jit_exit_groups);
73 if (!zend_jit_exit_groups) {
74 return FAILURE;
75 }
76 }
77
78 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
79 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
80
81 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
82 if (JIT_G(exit_counters) == NULL) {
83 return FAILURE;
84 }
85
86 return SUCCESS;
87 }
88
zend_jit_trace_allocate_exit_group(uint32_t n)89 static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
90 {
91 dasm_State* dasm_state = NULL;
92 const void *entry;
93 char name[32];
94
95 dasm_init(&dasm_state, DASM_MAXSECTION);
96 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
97 dasm_setup(&dasm_state, dasm_actions);
98 zend_jit_align_stub(&dasm_state);
99 zend_jit_trace_exit_group_stub(&dasm_state, n);
100
101 sprintf(name, "jit$$trace_exit_%d", n);
102 entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE);
103 dasm_free(&dasm_state);
104
105 #ifdef HAVE_DISASM
106 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
107 uint32_t i;
108
109 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
110 sprintf(name, "jit$$trace_exit_%d", n + i);
111 zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
112 }
113 }
114 #endif
115
116 return entry;
117 }
118
zend_jit_trace_allocate_exit_point(uint32_t n)119 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
120 {
121 const void *group = NULL;
122
123 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
124 return NULL;
125 }
126 do {
127 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
128 if (!group) {
129 return NULL;
130 }
131 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
132 group;
133 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
134 } while (n >= ZEND_JIT_EXIT_NUM);
135 return (const void*)
136 ((const char*)group +
137 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
138 }
139
zend_jit_trace_get_exit_addr(uint32_t n)140 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
141 {
142 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
143 return zend_jit_trace_allocate_exit_point(n);
144 }
145 return (const void*)
146 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
147 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
148 }
149
150 #if ZEND_JIT_TARGET_ARM64
zend_jit_get_current_trace_info(void)151 static zend_jit_trace_info *zend_jit_get_current_trace_info(void)
152 {
153 return &zend_jit_traces[ZEND_JIT_TRACE_NUM];
154 }
155
zend_jit_trace_find_exit_point(const void * addr)156 static uint32_t zend_jit_trace_find_exit_point(const void* addr)
157 {
158 uint32_t n = ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP;
159 uint32_t i;
160
161 for (i = 0; i < n; i++) {
162 if ((const char*)addr >= (const char*)zend_jit_exit_groups[i]
163 && (const char*)addr < (const char*)zend_jit_exit_groups[i] +
164 (ZEND_JIT_EXIT_POINTS_PER_GROUP * ZEND_JIT_EXIT_POINTS_SPACING)) {
165 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
166 ((const char*)addr - (const char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING;
167 }
168 }
169 return (uint32_t)-1;
170 }
171 #endif
172
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)173 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
174 {
175 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
176 uint32_t exit_point;
177 const zend_op_array *op_array;
178 uint32_t stack_offset = (uint32_t)-1;
179 uint32_t stack_size;
180 zend_jit_trace_stack *stack = NULL;
181
182 if (delayed_call_chain) {
183 assert(to_opline != NULL); /* CALL and IP share the same register */
184 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
185 }
186 if (JIT_G(current_frame)) {
187 op_array = &JIT_G(current_frame)->func->op_array;
188 stack_size = op_array->last_var + op_array->T;
189 if (stack_size) {
190 stack = JIT_G(current_frame)->stack;
191 do {
192 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
193 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
194 || STACK_REG(stack, stack_size-1) != ZREG_NONE) {
195 break;
196 }
197 stack_size--;
198 } while (stack_size);
199 }
200 } else {
201 op_array = NULL;
202 stack_size = 0;
203 }
204
205 /* Try to reuse exit points */
206 if (to_opline != NULL && t->exit_count > 0) {
207 uint32_t i = t->exit_count;
208
209 do {
210 i--;
211 if (stack_size == 0
212 || (t->exit_info[i].stack_size >= stack_size
213 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
214 stack_offset = t->exit_info[i].stack_offset;
215 if (t->exit_info[i].opline == to_opline
216 && t->exit_info[i].flags == flags
217 && t->exit_info[i].stack_size == stack_size) {
218 return i;
219 }
220 }
221 } while (i > 0);
222 }
223
224 exit_point = t->exit_count;
225 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
226 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
227 stack_offset = t->stack_map_size;
228 t->stack_map_size += stack_size;
229 // TODO: reduce number of reallocations ???
230 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
231 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
232 }
233 t->exit_count++;
234 t->exit_info[exit_point].opline = to_opline;
235 t->exit_info[exit_point].op_array = op_array;
236 t->exit_info[exit_point].flags = flags;
237 t->exit_info[exit_point].stack_size = stack_size;
238 t->exit_info[exit_point].stack_offset = stack_offset;
239 }
240
241 return exit_point;
242 }
243
zend_jit_trace_add_code(const void * start,uint32_t size)244 static void zend_jit_trace_add_code(const void *start, uint32_t size)
245 {
246 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
247
248 t->code_start = start;
249 t->code_size = size;
250 }
251
252 /**
253 * Locate a trace in the #zend_jit_traces array with the specified
254 * #code_start address.
255 *
256 * @return the #zend_jit_traces index or 0 if no such #code_start
257 * address was found
258 */
zend_jit_find_trace(const void * addr)259 static uint32_t zend_jit_find_trace(const void *addr)
260 {
261 uint32_t i;
262
263 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
264 if (zend_jit_traces[i].code_start == addr) {
265 return i;
266 }
267 }
268 return 0;
269 }
270
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)271 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
272 {
273 smart_str buf = {0};
274
275 smart_str_appends(&buf, TRACE_PREFIX);
276 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
277 smart_str_appendc(&buf, '$');
278 if (op_array->function_name) {
279 if (op_array->scope) {
280 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
281 smart_str_appends(&buf, "::");
282 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
283 } else {
284 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
285 }
286 } else if (op_array->filename) {
287 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
288 }
289 smart_str_appendc(&buf, '$');
290 smart_str_append_long(&buf, (zend_long)lineno);
291 smart_str_0(&buf);
292 return buf.s;
293 }
294
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)295 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
296 {
297 switch (opline->opcode) {
298 case ZEND_IS_IDENTICAL:
299 case ZEND_IS_NOT_IDENTICAL:
300 case ZEND_IS_EQUAL:
301 case ZEND_IS_NOT_EQUAL:
302 case ZEND_IS_SMALLER:
303 case ZEND_IS_SMALLER_OR_EQUAL:
304 case ZEND_CASE:
305 case ZEND_CASE_STRICT:
306 case ZEND_ISSET_ISEMPTY_CV:
307 case ZEND_ISSET_ISEMPTY_VAR:
308 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
309 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
310 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
311 case ZEND_INSTANCEOF:
312 case ZEND_TYPE_CHECK:
313 case ZEND_DEFINED:
314 case ZEND_IN_ARRAY:
315 case ZEND_ARRAY_KEY_EXISTS:
316 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
317 /* smart branch */
318 return 1;
319 }
320 break;
321 case ZEND_JMPZ:
322 case ZEND_JMPNZ:
323 case ZEND_JMPZ_EX:
324 case ZEND_JMPNZ_EX:
325 case ZEND_JMP_SET:
326 case ZEND_COALESCE:
327 case ZEND_JMP_NULL:
328 case ZEND_FE_RESET_R:
329 case ZEND_FE_RESET_RW:
330 case ZEND_ASSERT_CHECK:
331 case ZEND_FE_FETCH_R:
332 case ZEND_FE_FETCH_RW:
333 case ZEND_SWITCH_LONG:
334 case ZEND_SWITCH_STRING:
335 case ZEND_MATCH:
336 case ZEND_BIND_INIT_STATIC_OR_JMP:
337 /* branch opcodes */
338 return 1;
339 case ZEND_NEW:
340 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
341 /* NEW may skip constructor without arguments */
342 return 1;
343 }
344 break;
345 case ZEND_CATCH:
346 case ZEND_FAST_CALL:
347 case ZEND_FAST_RET:
348 case ZEND_GENERATOR_CREATE:
349 case ZEND_GENERATOR_RETURN:
350 case ZEND_EXIT:
351 case ZEND_YIELD:
352 case ZEND_YIELD_FROM:
353 case ZEND_INCLUDE_OR_EVAL:
354 case ZEND_MATCH_ERROR:
355 /* unsupported */
356 return 1;
357 case ZEND_DO_FCALL:
358 /* potentially polymorphic call */
359 return 1;
360 #if 0
361 case ZEND_DO_UCALL:
362 case ZEND_DO_FCALL_BY_NAME:
363 /* monomorphic call */
364 // TODO: recompilation may change target ???
365 return 0;
366 #endif
367 case ZEND_RETURN_BY_REF:
368 case ZEND_RETURN:
369 /* return */
370 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
371 default:
372 break;
373 }
374 return 0;
375 }
376
zend_jit_trace_type_to_info_ex(uint8_t type,uint32_t info)377 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(uint8_t type, uint32_t info)
378 {
379 if (type == IS_UNKNOWN) {
380 return info;
381 }
382 ZEND_ASSERT(info & (1 << type));
383 if (type < IS_STRING) {
384 return (1 << type);
385 } else if (type != IS_ARRAY) {
386 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
387 } else {
388 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
389 }
390 }
391
zend_jit_trace_type_to_info(uint8_t type)392 static zend_always_inline uint32_t zend_jit_trace_type_to_info(uint8_t type)
393 {
394 return zend_jit_trace_type_to_info_ex(type, -1);
395 }
396
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)397 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
398 {
399 if (var >= op_array->last_var) {
400 return NO_ALIAS;
401 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
402 return SYMTABLE_ALIAS;
403 } else if (ssa->vars) {
404 return ssa->vars[var].alias;
405 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
406 return HTTP_RESPONSE_HEADER_ALIAS;
407 }
408 return NO_ALIAS;
409 }
410
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)411 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
412 int ssa_var,
413 uint8_t op_type)
414 {
415 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
416
417 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
418 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
419 info->type |= MAY_BE_GUARD;
420 } else {
421 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
422 }
423 }
424 }
425
426 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
427 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
428 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
429 } \
430 } while (0)
431
432 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
433 if (op_type != IS_UNKNOWN) { \
434 if ((op_info & MAY_BE_GUARD) != 0) { \
435 if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \
436 goto jit_failure; \
437 } \
438 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
439 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
440 op_info = zend_jit_trace_type_to_info(op_type); \
441 } else { \
442 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
443 op_info &= ~MAY_BE_GUARD; \
444 ssa->var_info[_ssa_var].type &= op_info; \
445 } \
446 } \
447 } \
448 } while (0)
449
450 #define ADD_OP1_TRACE_GUARD() \
451 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
452 #define ADD_OP2_TRACE_GUARD() \
453 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
454 #define ADD_OP1_DATA_TRACE_GUARD() \
455 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
456
457 #define CHECK_OP1_TRACE_TYPE() \
458 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
459 #define CHECK_OP2_TRACE_TYPE() \
460 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
461 #define CHECK_OP1_DATA_TRACE_TYPE() \
462 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
463
zend_jit_trace_frame_size(const zend_op_array * op_array)464 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
465 {
466 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
467 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
468 } else if (op_array) {
469 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
470 } else {
471 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
472 }
473 }
474
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)475 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
476 {
477 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
478 }
479
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)480 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
481 {
482 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
483 }
484
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,uint8_t type)485 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type)
486 {
487 zend_jit_trace_stack *stack = call->stack;
488 const zend_op_array *op_array = &call->func->op_array;
489 uint32_t arg_num = opline->op2.num;
490
491 if (arg_num > op_array->num_args) {
492 return;
493 }
494 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
495 zend_arg_info *arg_info;
496
497 ZEND_ASSERT(arg_num <= op_array->num_args);
498 arg_info = &op_array->arg_info[arg_num-1];
499
500 if (ZEND_TYPE_IS_SET(arg_info->type)) {
501 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
502 return;
503 }
504 }
505 }
506 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
507 }
508
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)509 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
510 {
511 if (func
512 && func->type == ZEND_INTERNAL_FUNCTION
513 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
514 && arg_num < func->internal_function.num_args) {
515 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
516
517 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
518 && ZEND_TYPE_IS_SET(arg_info->type)
519 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
520 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
521 && JIT_G(current_frame)
522 && JIT_G(current_frame)->call
523 && JIT_G(current_frame)->call->func) {
524 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
525
526 if (type != IS_UNKNOWN
527 && type < IS_STRING
528 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
529 return 0;
530 }
531 }
532 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
533 const zend_op *opline = call_info->arg_info[arg_num].opline;
534
535 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
536 zval *zv = RT_CONSTANT(opline, opline->op1);
537
538 if (!Z_REFCOUNTED_P(zv)) {
539 uint32_t type = Z_TYPE_P(zv);
540
541 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
542 if (type != IS_ARRAY
543 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
544 return 0;
545 }
546 }
547 }
548 }
549 }
550 }
551
552 return 1;
553 }
554
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)555 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
556 {
557 zend_jit_op_array_trace_extension *jit_extension;
558 zend_ssa *ssa;
559
560 jit_extension =
561 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
562 jit_extension->func_info.num = 0;
563 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
564 | ZEND_FUNC_JIT_ON_PROF_REQUEST
565 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
566 | ZEND_FUNC_JIT_ON_HOT_TRACE;
567 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
568 ssa = &jit_extension->func_info.ssa;
569
570 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
571 do {
572 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
573 break;
574 }
575
576 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
577 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info);
578 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
579 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
580 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
581 }
582 }
583
584 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
585 break;
586 }
587
588 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
589 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
590 }
591 return ssa;
592 } while (0);
593 }
594
595 memset(ssa, 0, sizeof(zend_ssa));
596 ssa->cfg.blocks_count = 1;
597
598 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
599 zend_cfg cfg;
600 void *checkpoint = zend_arena_checkpoint(CG(arena));
601
602 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
603 ssa->cfg.flags = cfg.flags;
604 } else{
605 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
606 }
607
608 /* TODO: move this to zend_cfg.c ? */
609 if (!op_array->function_name) {
610 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
611 }
612
613 zend_arena_release(&CG(arena), checkpoint);
614 }
615
616 return ssa;
617 }
618
619 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
620 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
621
zend_jit_trace_op_len(const zend_op * opline)622 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
623 {
624 int len;
625
626 switch (opline->opcode) {
627 case ZEND_ASSIGN_DIM:
628 case ZEND_ASSIGN_OBJ:
629 case ZEND_ASSIGN_STATIC_PROP:
630 case ZEND_ASSIGN_DIM_OP:
631 case ZEND_ASSIGN_OBJ_OP:
632 case ZEND_ASSIGN_STATIC_PROP_OP:
633 case ZEND_ASSIGN_OBJ_REF:
634 case ZEND_ASSIGN_STATIC_PROP_REF:
635 return 2; /* OP_DATA */
636 case ZEND_RECV_INIT:
637 len = 1;
638 opline++;
639 while (opline->opcode == ZEND_RECV_INIT) {
640 len++;
641 opline++;
642 }
643 return len;
644 case ZEND_BIND_GLOBAL:
645 len = 1;
646 opline++;
647 while (opline->opcode == ZEND_BIND_GLOBAL) {
648 len++;
649 opline++;
650 }
651 return len;
652 // case ZEND_IS_IDENTICAL:
653 // case ZEND_IS_NOT_IDENTICAL:
654 // case ZEND_IS_EQUAL:
655 // case ZEND_IS_NOT_EQUAL:
656 // case ZEND_IS_SMALLER:
657 // case ZEND_IS_SMALLER_OR_EQUAL:
658 // case ZEND_CASE:
659 // case ZEND_ISSET_ISEMPTY_CV:
660 // case ZEND_ISSET_ISEMPTY_VAR:
661 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
662 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
663 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
664 // case ZEND_INSTANCEOF:
665 // case ZEND_TYPE_CHECK:
666 // case ZEND_DEFINED:
667 // case ZEND_IN_ARRAY:
668 // case ZEND_ARRAY_KEY_EXISTS:
669 default:
670 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
671 return 2; /* JMPZ/JMPNZ */
672 }
673 return 1;
674 }
675 }
676
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)677 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
678 {
679 const zend_op_array *op_array;
680 zend_jit_trace_rec *p;
681 int k, vars_count;
682 zend_bitset use, def;
683 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
684 uint32_t set_size;
685 zend_ssa_phi *prev = NULL;
686 int level = 0;
687 ALLOCA_FLAG(use_heap);
688
689 op_array = trace_buffer->op_array;
690 set_size = zend_bitset_len(op_array->last_var + op_array->T);
691 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
692 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
693 def = use + set_size;
694 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
695 for (;;p++) {
696 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
697 const zend_op *opline = p->opline;
698 int len;
699
700 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
701 len = zend_jit_trace_op_len(opline);
702 while (len > 1) {
703 opline++;
704 if (opline->opcode != ZEND_OP_DATA) {
705 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
706 }
707 len--;
708 }
709 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
710 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
711 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
712 level++;
713 } else if (p->op == ZEND_JIT_TRACE_BACK) {
714 if (level == 0) {
715 // Phi for recursive calls and returns are not supported yet ???
716 assert(0);
717 } else {
718 level--;
719 }
720 } else if (p->op == ZEND_JIT_TRACE_END) {
721 break;
722 }
723 }
724
725 zend_bitset_intersection(use, def, set_size);
726
727 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
728 vars_count = op_array->last_var;
729 } else {
730 vars_count = op_array->last_var + op_array->T;
731 }
732 for (k = 0; k < vars_count; k++) {
733 if (zend_bitset_in(use, k)) {
734 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
735 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
736 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
737 sizeof(void*) * 2);
738 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
739 phi->sources[0] = STACK_VAR(stack, k);
740 phi->sources[1] = -1;
741 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
742 phi->pi = -1;
743 phi->var = k;
744 phi->ssa_var = ssa_vars_count;
745 SET_STACK_VAR(stack, k, ssa_vars_count);
746 ssa_vars_count++;
747 phi->block = 1;
748 if (prev) {
749 prev->next = phi;
750 } else {
751 tssa->blocks[1].phis = phi;
752 }
753 prev = phi;
754 }
755 }
756
757 free_alloca(use, use_heap);
758
759 return ssa_vars_count;
760 }
761
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)762 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
763 {
764 zend_ssa_phi *prev = NULL;
765 const zend_op_array *op_array = trace_buffer->op_array;
766 const zend_op *opline = trace_buffer[1].opline;
767 int count = opline - op_array->opcodes;
768 int i;
769
770 for(i = 0; i < count; i++) {
771 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
772 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
773 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
774 sizeof(void*) * 2);
775 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
776 phi->sources[0] = STACK_VAR(stack, i);
777 phi->sources[1] = -1;
778 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
779 phi->pi = -1;
780 phi->var = i;
781 phi->ssa_var = ssa_vars_count;
782 SET_STACK_VAR(stack, i, ssa_vars_count);
783 ssa_vars_count++;
784 phi->block = 1;
785 if (prev) {
786 prev->next = phi;
787 } else {
788 tssa->blocks[1].phis = phi;
789 }
790 prev = phi;
791 }
792 return ssa_vars_count;
793 }
794
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)795 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
796 {
797 const zend_op *opline = trace_buffer[1].opline - 1;
798 int i;
799
800 if (RETURN_VALUE_USED(opline)) {
801 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
802 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
803 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
804 sizeof(void*) * 2);
805
806 i = EX_VAR_TO_NUM(opline->result.var);
807 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
808 phi->sources[0] = STACK_VAR(stack, i);
809 phi->sources[1] = -1;
810 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
811 phi->pi = -1;
812 phi->var = i;
813 phi->ssa_var = ssa_vars_count;
814 SET_STACK_VAR(stack, i, ssa_vars_count);
815 ssa_vars_count++;
816 phi->block = 1;
817 tssa->blocks[1].phis = phi;
818 }
819 return ssa_vars_count;
820 }
821
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)822 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
823 {
824 int var, use;
825 zend_ssa_op *op;
826 zend_ssa_var_info *info;
827 unsigned int no_val;
828 zend_ssa_alias_kind alias;
829
830 if (tssa->vars[ssa_var].phi_use_chain) {
831 // TODO: this may be incorrect ???
832 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
833 } else {
834 var = ssa_var;
835 }
836 use = tssa->vars[var].use_chain;
837 if (use >= 0) {
838 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
839 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
840 if (tssa->ops[use].op1_use == var) {
841 no_val = ssa->vars[op->op1_use].no_val;
842 alias = ssa->vars[op->op1_use].alias;
843 info = ssa->var_info + op->op1_use;
844 } else if (tssa->ops[use].op2_use == var) {
845 no_val = ssa->vars[op->op2_use].no_val;
846 alias = ssa->vars[op->op2_use].alias;
847 info = ssa->var_info + op->op2_use;
848 } else if (tssa->ops[use].result_use == var) {
849 no_val = ssa->vars[op->result_use].no_val;
850 alias = ssa->vars[op->result_use].alias;
851 info = ssa->var_info + op->result_use;
852 } else {
853 assert(0);
854 return 0;
855 }
856 tssa->vars[ssa_var].no_val = no_val;
857 tssa->vars[ssa_var].alias = alias;
858 memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info));
859 return 1;
860 }
861 return 0;
862 }
863
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)864 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
865 {
866 zend_ssa_range tmp;
867 int def = tssa->vars[ssa_var].definition;
868
869 if (tssa->vars[ssa_var].alias == NO_ALIAS
870 && zend_inference_propagate_range(op_array, tssa, tssa_opcodes[def], &tssa->ops[def], ssa_var, &tmp)) {
871 tssa->var_info[ssa_var].range.min = tmp.min;
872 tssa->var_info[ssa_var].range.max = tmp.max;
873 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
874 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
875 tssa->var_info[ssa_var].has_range = 1;
876 }
877 }
878
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)879 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
880 {
881 int def;
882 zend_ssa_op *op;
883 zend_ssa_var_info *info;
884 unsigned int no_val;
885 zend_ssa_alias_kind alias;
886
887 def = tssa->vars[ssa_var].definition;
888 if (def >= 0) {
889 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
890 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
891 if (tssa->ops[def].op1_def == ssa_var) {
892 no_val = ssa->vars[op->op1_def].no_val;
893 alias = ssa->vars[op->op1_def].alias;
894 info = ssa->var_info + op->op1_def;
895 } else if (tssa->ops[def].op2_def == ssa_var) {
896 no_val = ssa->vars[op->op2_def].no_val;
897 alias = ssa->vars[op->op2_def].alias;
898 info = ssa->var_info + op->op2_def;
899 } else if (tssa->ops[def].result_def == ssa_var) {
900 no_val = ssa->vars[op->result_def].no_val;
901 alias = ssa->vars[op->result_def].alias;
902 info = ssa->var_info + op->result_def;
903 } else {
904 assert(0);
905 return;
906 }
907
908 tssa->vars[ssa_var].no_val = no_val;
909 tssa->vars[ssa_var].alias = alias;
910
911 if (!(info->type & MAY_BE_REF)) {
912 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
913 }
914
915 if (info->has_range) {
916 if (tssa->var_info[ssa_var].has_range) {
917 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
918 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
919 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
920 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
921 } else {
922 tssa->var_info[ssa_var].has_range = 1;
923 tssa->var_info[ssa_var].range = info->range;
924 }
925 }
926 }
927 }
928
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)929 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
930 {
931 int def;
932 zend_ssa_op *op;
933 zend_ssa_var_info *info;
934
935 def = tssa->vars[ssa_var].definition;
936 if (def >= 0) {
937 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
938 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
939 if (tssa->ops[def].op1_def == ssa_var) {
940 info = ssa->var_info + op->op1_def;
941 } else if (tssa->ops[def].op2_def == ssa_var) {
942 info = ssa->var_info + op->op2_def;
943 } else if (tssa->ops[def].result_def == ssa_var) {
944 info = ssa->var_info + op->result_def;
945 } else {
946 assert(0);
947 return 0;
948 }
949 tssa->var_info[ssa_var].type &= info->type;
950 if (info->ce) {
951 if (tssa->var_info[ssa_var].ce) {
952 if (tssa->var_info[ssa_var].ce != info->ce) {
953 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
954 /* everything fine */
955 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
956 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
957 } else {
958 // TODO: classes may implement the same interface ???
959 //ZEND_UNREACHABLE();
960 }
961 }
962 tssa->var_info[ssa_var].is_instanceof =
963 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
964 } else {
965 tssa->var_info[ssa_var].ce = info->ce;
966 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
967 }
968 }
969 if (info->has_range) {
970 if (tssa->var_info[ssa_var].has_range) {
971 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
972 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
973 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
974 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
975 } else {
976 tssa->var_info[ssa_var].has_range = 1;
977 tssa->var_info[ssa_var].range = info->range;
978 }
979 }
980 return 1;
981 }
982 return 0;
983 }
984
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)985 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
986 {
987 while (1) {
988 if (p->op == ZEND_JIT_TRACE_VM) {
989 if (p->opline->opcode == ZEND_DO_UCALL
990 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
991 || p->opline->opcode == ZEND_DO_FCALL) {
992 if (p->opline->result_type != IS_UNUSED) {
993 return ssa_op->result_def;
994 }
995 }
996 return -1;
997 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
998 /*skip */
999 } else {
1000 return -1;
1001 }
1002 p--;
1003 }
1004 }
1005
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1006 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1007 {
1008 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1009 p--;
1010 while (1) {
1011 if (p->op == ZEND_JIT_TRACE_VM) {
1012 if (p->opline->opcode == ZEND_INIT_FCALL
1013 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1014 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1015 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1016 || p->opline->opcode == ZEND_INIT_USER_CALL
1017 || p->opline->opcode == ZEND_NEW
1018 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1019 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
1020 return p->opline;
1021 }
1022 return NULL;
1023 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1024 /*skip */
1025 } else {
1026 return NULL;
1027 }
1028 p--;
1029 }
1030 } else {
1031 const zend_op *opline = NULL;
1032 int call_level = 0;
1033
1034 p++;
1035 while (1) {
1036 if (p->op == ZEND_JIT_TRACE_VM) {
1037 opline = p->opline;
1038 break;
1039 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1040 call_level++;
1041 /*skip */
1042 } else {
1043 return NULL;
1044 }
1045 p--;
1046 }
1047 if (opline) {
1048 while (opline > op_array->opcodes) {
1049 opline--;
1050 switch (opline->opcode) {
1051 case ZEND_INIT_FCALL:
1052 case ZEND_INIT_FCALL_BY_NAME:
1053 case ZEND_INIT_NS_FCALL_BY_NAME:
1054 case ZEND_INIT_METHOD_CALL:
1055 case ZEND_INIT_DYNAMIC_CALL:
1056 case ZEND_INIT_STATIC_METHOD_CALL:
1057 case ZEND_INIT_USER_CALL:
1058 case ZEND_NEW:
1059 if (call_level == 0) {
1060 return opline;
1061 }
1062 call_level--;
1063 break;
1064 case ZEND_DO_FCALL:
1065 case ZEND_DO_ICALL:
1066 case ZEND_DO_UCALL:
1067 case ZEND_DO_FCALL_BY_NAME:
1068 case ZEND_CALLABLE_CONVERT:
1069 call_level++;
1070 break;
1071 }
1072 }
1073 }
1074 }
1075 return NULL;
1076 }
1077
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1078 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1079 {
1080 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1081 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1082 int idx = tssa->vars[var].definition;
1083
1084 if (idx >= 0) {
1085 if (tssa->ops[idx].op1_def == var) {
1086 const zend_op *opline = ssa_opcodes[idx];
1087 if (opline->opcode == ZEND_PRE_DEC
1088 || opline->opcode == ZEND_PRE_INC
1089 || opline->opcode == ZEND_POST_DEC
1090 || opline->opcode == ZEND_POST_INC) {
1091 if (tssa->ops[idx].op1_use >= 0
1092 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1093 return 0;
1094 }
1095 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1096 return 0;
1097 }
1098 return 1;
1099 } else if (opline->opcode == ZEND_ASSIGN_OP
1100 && (opline->extended_value == ZEND_ADD
1101 || opline->extended_value == ZEND_SUB
1102 || opline->extended_value == ZEND_MUL)) {
1103 if ((opline->op2_type & (IS_VAR|IS_CV))
1104 && tssa->ops[idx].op2_use >= 0
1105 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1106 return 0;
1107 }
1108 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1109 return 0;
1110 }
1111 if (opline->op2_type == IS_CONST) {
1112 zval *zv = RT_CONSTANT(opline, opline->op2);
1113 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1114 return 0;
1115 }
1116 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1117 return 0;
1118 }
1119 return 1;
1120 }
1121 }
1122 if (tssa->ops[idx].result_def == var) {
1123 const zend_op *opline = ssa_opcodes[idx];
1124 if (opline->opcode == ZEND_ADD
1125 || opline->opcode == ZEND_SUB
1126 || opline->opcode == ZEND_MUL) {
1127 if ((opline->op1_type & (IS_VAR|IS_CV))
1128 && tssa->ops[idx].op1_use >= 0
1129 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1130 return 0;
1131 }
1132 if ((opline->op2_type & (IS_VAR|IS_CV))
1133 && tssa->ops[idx].op2_use >= 0
1134 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1135 return 0;
1136 }
1137 if (opline->op1_type == IS_CONST) {
1138 zval *zv = RT_CONSTANT(opline, opline->op1);
1139 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1140 return 0;
1141 }
1142 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1143 return 0;
1144 }
1145 if (opline->op2_type == IS_CONST) {
1146 zval *zv = RT_CONSTANT(opline, opline->op2);
1147 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1148 return 0;
1149 }
1150 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1151 return 0;
1152 }
1153 return 1;
1154 } else if (opline->opcode == ZEND_PRE_DEC
1155 || opline->opcode == ZEND_PRE_INC
1156 || opline->opcode == ZEND_POST_DEC
1157 || opline->opcode == ZEND_POST_INC) {
1158 if ((opline->op1_type & (IS_VAR|IS_CV))
1159 && tssa->ops[idx].op1_use >= 0
1160 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1161 return 0;
1162 }
1163 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1164 return 0;
1165 }
1166 return 1;
1167 }
1168 }
1169 }
1170 }
1171 return 0;
1172 }
1173
1174 typedef struct _zend_tssa {
1175 zend_ssa ssa;
1176 const zend_op **tssa_opcodes;
1177 int used_stack;
1178 } zend_tssa;
1179
1180 static const zend_op _nop_opcode = {0};
1181
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1182 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1183 {
1184 zend_ssa *tssa;
1185 zend_ssa_op *ssa_ops, *op;
1186 zend_ssa_var *ssa_vars;
1187 zend_ssa_var_info *ssa_var_info;
1188 const zend_op_array *op_array;
1189 const zend_op *opline;
1190 const zend_op **ssa_opcodes;
1191 zend_jit_trace_rec *p;
1192 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1193 zend_jit_trace_stack *stack;
1194 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1195 uint32_t optimization_level = 0;
1196 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1197 size_t frame_size, stack_top, stack_size, stack_bottom;
1198 zend_jit_op_array_trace_extension *jit_extension;
1199 zend_ssa *ssa;
1200 zend_jit_trace_stack_frame *frame, *top, *call;
1201 zend_ssa_var_info return_value_info;
1202
1203 /* 1. Count number of TSSA opcodes;
1204 * Count number of activation frames;
1205 * Calculate size of abstract stack;
1206 * Construct regular SSA for involved op_array */
1207 op_array = trace_buffer->op_array;
1208 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1209 stack_bottom = 0;
1210 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1211 ssa_ops_count = 0;
1212 call_level = 0;
1213 level = 0;
1214 num_op_arrays = 0;
1215 /* Remember op_array to cleanup */
1216 op_arrays[num_op_arrays++] = op_array;
1217 /* Build SSA */
1218 ssa = zend_jit_trace_build_ssa(op_array, script);
1219 for (;;p++) {
1220 if (p->op == ZEND_JIT_TRACE_VM) {
1221 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1222 const zend_op *opline = p->opline;
1223
1224 switch (opline->opcode) {
1225 case ZEND_INCLUDE_OR_EVAL:
1226 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1227 break;
1228 case ZEND_FETCH_R:
1229 case ZEND_FETCH_W:
1230 case ZEND_FETCH_RW:
1231 case ZEND_FETCH_FUNC_ARG:
1232 case ZEND_FETCH_IS:
1233 case ZEND_FETCH_UNSET:
1234 case ZEND_UNSET_VAR:
1235 case ZEND_ISSET_ISEMPTY_VAR:
1236 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1237 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1238 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1239 !op_array->function_name) {
1240 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1241 }
1242 break;
1243 }
1244 }
1245 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1246 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1247 call_level++;
1248 stack_top += zend_jit_trace_frame_size(p->op_array);
1249 if (stack_top > stack_size) {
1250 stack_size = stack_top;
1251 }
1252 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1253 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1254 if (p->func != (zend_function*)&zend_pass_function
1255 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1256 || zend_string_equals_literal(p->func->common.function_name, "compact")
1257 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1258 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1259 }
1260 }
1261 frame_size = zend_jit_trace_frame_size(p->op_array);
1262 if (call_level == 0) {
1263 if (stack_top + frame_size > stack_size) {
1264 stack_size = stack_top + frame_size;
1265 }
1266 } else {
1267 call_level--;
1268 stack_top -= frame_size;
1269 }
1270 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1271 op_array = p->op_array;
1272 if (call_level == 0) {
1273 stack_top += zend_jit_trace_frame_size(op_array);
1274 if (stack_top > stack_size) {
1275 stack_size = stack_top;
1276 }
1277 } else {
1278 call_level--;
1279 }
1280 level++;
1281 jit_extension =
1282 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1283 ssa = &jit_extension->func_info.ssa;
1284 if (ssa->cfg.blocks_count) {
1285 /* pass */
1286 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1287 /* Too many functions in single trace */
1288 *num_op_arrays_ptr = num_op_arrays;
1289 return NULL;
1290 } else {
1291 /* Remember op_array to cleanup */
1292 op_arrays[num_op_arrays++] = op_array;
1293 /* Build SSA */
1294 ssa = zend_jit_trace_build_ssa(op_array, script);
1295 }
1296 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1297 if (level == 0) {
1298 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1299 jit_extension =
1300 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1301 ssa = &jit_extension->func_info.ssa;
1302 if (ssa->cfg.blocks_count) {
1303 /* pass */
1304 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1305 /* Too many functions in single trace */
1306 *num_op_arrays_ptr = num_op_arrays;
1307 return NULL;
1308 } else {
1309 /* Remember op_array to cleanup */
1310 op_arrays[num_op_arrays++] = op_array;
1311 /* Build SSA */
1312 ssa = zend_jit_trace_build_ssa(op_array, script);
1313 }
1314 } else {
1315 stack_top -= zend_jit_trace_frame_size(op_array);
1316 level--;
1317 }
1318 op_array = p->op_array;
1319 } else if (p->op == ZEND_JIT_TRACE_END) {
1320 break;
1321 }
1322 }
1323 *num_op_arrays_ptr = num_op_arrays;
1324
1325 /* Allocate space for abstract stack */
1326 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1327
1328 /* 2. Construct TSSA */
1329 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1330 tssa->cfg.flags = ZEND_SSA_TSSA;
1331 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1332 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1333 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1334
1335 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1336 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1337 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1338 tssa->cfg.blocks_count = 2;
1339 tssa->cfg.edges_count = 2;
1340
1341 tssa->cfg.predecessors[0] = 0;
1342 tssa->cfg.predecessors[1] = 1;
1343
1344 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1345 tssa->cfg.blocks[0].successors_count = 1;
1346 tssa->cfg.blocks[0].predecessors_count = 0;
1347 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1348 tssa->cfg.blocks[0].successors[0] = 1;
1349
1350 tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1351 tssa->cfg.blocks[1].successors_count = 1;
1352 tssa->cfg.blocks[1].predecessors_count = 2;
1353 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1354 tssa->cfg.blocks[1].successors[1] = 1;
1355 } else {
1356 tssa->cfg.blocks_count = 1;
1357 tssa->cfg.edges_count = 0;
1358
1359 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1360 tssa->cfg.blocks[0].successors_count = 0;
1361 tssa->cfg.blocks[0].predecessors_count = 0;
1362 }
1363 ((zend_tssa*)tssa)->used_stack = -1;
1364
1365 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1366 return tssa;
1367 }
1368
1369 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1370 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1371 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1372 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1373 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1374
1375 op_array = trace_buffer->op_array;
1376 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1377 ssa_vars_count = op_array->last_var;
1378 } else {
1379 ssa_vars_count = op_array->last_var + op_array->T;
1380 }
1381 stack = frame->stack;
1382 for (i = 0; i < ssa_vars_count; i++) {
1383 SET_STACK_VAR(stack, i, i);
1384 }
1385
1386 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1387 // TODO: For tracing, it's possible, to create pseudo Phi functions
1388 // at the end of loop, without this additional pass (like LuaJIT) ???
1389 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1390 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1391 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1392 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1393 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1394 }
1395
1396 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1397 idx = 0;
1398 level = 0;
1399 for (;;p++) {
1400 if (p->op == ZEND_JIT_TRACE_VM) {
1401 opline = p->opline;
1402 ssa_opcodes[idx] = opline;
1403 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1404 idx++;
1405 len = zend_jit_trace_op_len(p->opline);
1406 while (len > 1) {
1407 opline++;
1408 ssa_opcodes[idx] = opline;
1409 if (opline->opcode != ZEND_OP_DATA) {
1410 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1411 }
1412 idx++;
1413 len--;
1414 }
1415 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1416 frame = zend_jit_trace_call_frame(frame, op_array);
1417 stack = frame->stack;
1418 op_array = p->op_array;
1419 level++;
1420 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1421 return NULL;
1422 }
1423 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1424 for (i = 0; i < op_array->last_var; i++) {
1425 SET_STACK_VAR(stack, i, ssa_vars_count++);
1426 }
1427 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1428 op_array = p->op_array;
1429 frame = zend_jit_trace_ret_frame(frame, op_array);
1430 stack = frame->stack;
1431 if (level == 0) {
1432 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1433 return NULL;
1434 }
1435 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1436 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1437 SET_STACK_VAR(stack, i, ssa_vars_count++);
1438 }
1439 } else {
1440 level--;
1441 }
1442 } else if (p->op == ZEND_JIT_TRACE_END) {
1443 break;
1444 }
1445 }
1446
1447 op_array = trace_buffer->op_array;
1448 tssa->vars_count = ssa_vars_count;
1449 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1450 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1451 vars_count = op_array->last_var;
1452 } else {
1453 vars_count = op_array->last_var + op_array->T;
1454 }
1455 i = 0;
1456 while (i < vars_count) {
1457 ssa_vars[i].var = i;
1458 ssa_vars[i].scc = -1;
1459 ssa_vars[i].definition = -1;
1460 ssa_vars[i].use_chain = -1;
1461 i++;
1462 }
1463 while (i < tssa->vars_count) {
1464 ssa_vars[i].var = -1;
1465 ssa_vars[i].scc = -1;
1466 ssa_vars[i].definition = -1;
1467 ssa_vars[i].use_chain = -1;
1468 i++;
1469 }
1470
1471 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1472 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1473 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1474 /* Update Phi sources */
1475 zend_ssa_phi *phi = tssa->blocks[1].phis;
1476
1477 while (phi) {
1478 phi->sources[1] = STACK_VAR(stack, phi->var);
1479 ssa_vars[phi->ssa_var].var = phi->var;
1480 ssa_vars[phi->ssa_var].definition_phi = phi;
1481 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1482 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1483 phi = phi->next;
1484 }
1485 }
1486
1487 /* 3. Compute use-def chains */
1488 idx = (ssa_ops_count - 1);
1489 op = ssa_ops + idx;
1490 while (idx >= 0) {
1491 opline = ssa_opcodes[idx];
1492 if (op->op1_use >= 0) {
1493 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1494 ssa_vars[op->op1_use].use_chain = idx;
1495 }
1496 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1497 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1498 ssa_vars[op->op2_use].use_chain = idx;
1499 }
1500 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1501 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1502 ssa_vars[op->result_use].use_chain = idx;
1503 }
1504 if (op->op1_def >= 0) {
1505 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1506 ssa_vars[op->op1_def].definition = idx;
1507 }
1508 if (op->op2_def >= 0) {
1509 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1510 ssa_vars[op->op2_def].definition = idx;
1511 }
1512 if (op->result_def >= 0) {
1513 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1514 ssa_vars[op->result_def].definition = idx;
1515 }
1516 op--;
1517 idx--;
1518 }
1519
1520 /* 4. Type inference */
1521 op_array = trace_buffer->op_array;
1522 jit_extension =
1523 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1524 ssa = &jit_extension->func_info.ssa;
1525
1526 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1527
1528 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1529 i = 0;
1530 while (i < op_array->last_var) {
1531 if (i < op_array->num_args) {
1532 if (ssa->var_info
1533 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1534 /* pass */
1535 } else {
1536 if (ssa->vars) {
1537 ssa_vars[i].no_val = ssa->vars[i].no_val;
1538 ssa_vars[i].alias = ssa->vars[i].alias;
1539 } else {
1540 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1541 }
1542 if (op_array->arg_info && i < trace_buffer[1].opline - op_array->opcodes) {
1543 zend_arg_info *arg_info = &op_array->arg_info[i];
1544 zend_class_entry *ce;
1545 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1546
1547 if (ZEND_ARG_SEND_MODE(arg_info)) {
1548 tmp |= MAY_BE_REF;
1549 }
1550 ssa_var_info[i].type = tmp;
1551 ssa_var_info[i].ce = ce;
1552 ssa_var_info[i].is_instanceof = 1;
1553 } else {
1554 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1555 }
1556 }
1557 } else {
1558 if (ssa->vars) {
1559 ssa_vars[i].no_val = ssa->vars[i].no_val;
1560 ssa_vars[i].alias = ssa->vars[i].alias;
1561 } else {
1562 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1563 }
1564 if (ssa_vars[i].alias == NO_ALIAS) {
1565 ssa_var_info[i].type = MAY_BE_UNDEF;
1566 } else {
1567 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1568 }
1569 }
1570 i++;
1571 }
1572 } else {
1573 int parent_vars_count = 0;
1574 zend_jit_trace_stack *parent_stack = NULL;
1575
1576 i = 0;
1577 if (parent_trace) {
1578 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1579 op_array->last_var + op_array->T);
1580 if (parent_vars_count) {
1581 parent_stack =
1582 zend_jit_traces[parent_trace].stack_map +
1583 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1584 }
1585 }
1586 while (i < op_array->last_var + op_array->T) {
1587 if (!ssa->var_info
1588 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1589 if (ssa->vars && i < ssa->vars_count) {
1590 ssa_vars[i].alias = ssa->vars[i].alias;
1591 } else {
1592 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1593 }
1594 if (i < op_array->last_var) {
1595 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1596 } else {
1597 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1598 }
1599 }
1600 if (i < parent_vars_count) {
1601 /* Initialize TSSA variable from parent trace */
1602 uint8_t op_type = STACK_TYPE(parent_stack, i);
1603
1604 if (op_type != IS_UNKNOWN) {
1605 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1606 if (!ssa_var_info[i].type
1607 && op_type == IS_UNDEF
1608 && i >= op_array->last_var) {
1609 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1610 ssa_var_info[i].type |= MAY_BE_UNDEF;
1611 }
1612 }
1613 }
1614 i++;
1615 }
1616 }
1617
1618 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1619 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1620 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1621 /* Propagate initial value through Phi functions */
1622 zend_ssa_phi *phi = tssa->blocks[1].phis;
1623
1624 while (phi) {
1625 if (!ssa->var_info
1626 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) {
1627 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1628 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1629 }
1630 phi = phi->next;
1631 }
1632 }
1633
1634 frame = JIT_G(current_frame);
1635 top = zend_jit_trace_call_frame(frame, op_array);
1636 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1637 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1638 frame->used_stack = 0;
1639 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1640 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
1641 }
1642 memset(&return_value_info, 0, sizeof(return_value_info));
1643
1644 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1645 max_used_stack = used_stack = 0;
1646 } else {
1647 max_used_stack = used_stack = -1;
1648 }
1649
1650 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1651 idx = 0;
1652 level = 0;
1653 opline = NULL;
1654 for (;;p++) {
1655 if (p->op == ZEND_JIT_TRACE_VM) {
1656 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1657 uint8_t val_type = IS_UNKNOWN;
1658 // zend_class_entry *op1_ce = NULL;
1659 zend_class_entry *op2_ce = NULL;
1660
1661 opline = p->opline;
1662
1663 op1_type = orig_op1_type = p->op1_type;
1664 op2_type = orig_op2_type = p->op2_type;
1665 op3_type = p->op3_type;
1666 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1667 op1_type = IS_UNKNOWN;
1668 }
1669 if (op1_type != IS_UNKNOWN) {
1670 op1_type &= ~IS_TRACE_PACKED;
1671 }
1672 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1673 op2_type = IS_UNKNOWN;
1674 }
1675 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1676 op3_type = IS_UNKNOWN;
1677 }
1678
1679 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1680 // op1_ce = (zend_class_entry*)(p+1)->ce;
1681 p++;
1682 }
1683 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1684 op2_ce = (zend_class_entry*)(p+1)->ce;
1685 p++;
1686 }
1687 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1688 val_type = (p+1)->op1_type;
1689 p++;
1690 }
1691
1692 switch (opline->opcode) {
1693 case ZEND_ASSIGN_OP:
1694 if (opline->extended_value == ZEND_POW
1695 || opline->extended_value == ZEND_DIV) {
1696 // TODO: check for division by zero ???
1697 break;
1698 }
1699 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1700 break;
1701 }
1702 ADD_OP1_TRACE_GUARD();
1703 ADD_OP2_TRACE_GUARD();
1704 break;
1705 case ZEND_ASSIGN_DIM_OP:
1706 if (opline->extended_value == ZEND_POW
1707 || opline->extended_value == ZEND_DIV) {
1708 // TODO: check for division by zero ???
1709 break;
1710 }
1711 if (opline->result_type != IS_UNUSED) {
1712 break;
1713 }
1714 if (op3_type != IS_UNKNOWN
1715 && !zend_jit_supported_binary_op(
1716 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1717 break;
1718 }
1719 ZEND_FALLTHROUGH;
1720 case ZEND_ASSIGN_DIM:
1721 if (opline->op1_type == IS_CV) {
1722 if ((opline+1)->op1_type == IS_CV
1723 && (opline+1)->op1.var == opline->op1.var) {
1724 /* skip $a[x] = $a; */
1725 break;
1726 }
1727 ADD_OP1_DATA_TRACE_GUARD();
1728 ADD_OP2_TRACE_GUARD();
1729 ADD_OP1_TRACE_GUARD();
1730 } else if (orig_op1_type != IS_UNKNOWN
1731 && (orig_op1_type & IS_TRACE_INDIRECT)
1732 && opline->result_type == IS_UNUSED) {
1733 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1734 ADD_OP1_DATA_TRACE_GUARD();
1735 }
1736 ADD_OP2_TRACE_GUARD();
1737 }
1738 if (op1_type == IS_ARRAY
1739 && ((opline->op2_type == IS_CONST
1740 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1741 || (opline->op2_type != IS_CONST
1742 && op2_type == IS_LONG))) {
1743
1744 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1745 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1746
1747 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1748 info->type |= MAY_BE_PACKED_GUARD;
1749 info->type &= ~MAY_BE_ARRAY_PACKED;
1750 }
1751 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1752 && val_type != IS_UNKNOWN
1753 && val_type != IS_UNDEF) {
1754 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1755
1756 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1757 info->type |= MAY_BE_PACKED_GUARD;
1758 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1759 }
1760 }
1761 }
1762 break;
1763 case ZEND_ASSIGN_OBJ_OP:
1764 if (opline->extended_value == ZEND_POW
1765 || opline->extended_value == ZEND_DIV) {
1766 // TODO: check for division by zero ???
1767 break;
1768 }
1769 if (opline->result_type != IS_UNUSED) {
1770 break;
1771 }
1772 ZEND_FALLTHROUGH;
1773 case ZEND_ASSIGN_OBJ:
1774 case ZEND_PRE_INC_OBJ:
1775 case ZEND_PRE_DEC_OBJ:
1776 case ZEND_POST_INC_OBJ:
1777 case ZEND_POST_DEC_OBJ:
1778 if (opline->op2_type != IS_CONST
1779 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1780 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1781 break;
1782 }
1783 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1784 if (opline->op1_type == IS_CV
1785 && (opline+1)->op1_type == IS_CV
1786 && (opline+1)->op1.var == opline->op1.var) {
1787 /* skip $a->prop += $a; */
1788 break;
1789 }
1790 ADD_OP1_DATA_TRACE_GUARD();
1791 }
1792 ADD_OP1_TRACE_GUARD();
1793 break;
1794 case ZEND_CONCAT:
1795 case ZEND_FAST_CONCAT:
1796 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1797 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1798 ADD_OP2_TRACE_GUARD();
1799 ADD_OP1_TRACE_GUARD();
1800 }
1801 break;
1802 case ZEND_IS_EQUAL:
1803 case ZEND_IS_NOT_EQUAL:
1804 case ZEND_IS_SMALLER:
1805 case ZEND_IS_SMALLER_OR_EQUAL:
1806 case ZEND_CASE:
1807 case ZEND_IS_IDENTICAL:
1808 case ZEND_IS_NOT_IDENTICAL:
1809 case ZEND_CASE_STRICT:
1810 case ZEND_BW_OR:
1811 case ZEND_BW_AND:
1812 case ZEND_BW_XOR:
1813 case ZEND_SL:
1814 case ZEND_SR:
1815 case ZEND_MOD:
1816 case ZEND_ADD:
1817 case ZEND_SUB:
1818 case ZEND_MUL:
1819 // case ZEND_DIV: // TODO: check for division by zero ???
1820 ADD_OP2_TRACE_GUARD();
1821 ZEND_FALLTHROUGH;
1822 case ZEND_ECHO:
1823 case ZEND_STRLEN:
1824 case ZEND_COUNT:
1825 case ZEND_QM_ASSIGN:
1826 case ZEND_FE_RESET_R:
1827 ADD_OP1_TRACE_GUARD();
1828 break;
1829 case ZEND_FE_FETCH_R:
1830 ADD_OP1_TRACE_GUARD();
1831 if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) {
1832
1833 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1834
1835 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1836 info->type |= MAY_BE_PACKED_GUARD;
1837 if (orig_op1_type & IS_TRACE_PACKED) {
1838 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1839 } else {
1840 info->type &= ~MAY_BE_ARRAY_PACKED;
1841 }
1842 }
1843 }
1844 break;
1845 case ZEND_VERIFY_RETURN_TYPE:
1846 if (opline->op1_type == IS_UNUSED) {
1847 /* Always throws */
1848 break;
1849 }
1850 if (opline->op1_type == IS_CONST) {
1851 /* TODO Different instruction format, has return value */
1852 break;
1853 }
1854 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1855 /* Not worth bothering with */
1856 break;
1857 }
1858 ADD_OP1_TRACE_GUARD();
1859 break;
1860 case ZEND_FETCH_DIM_FUNC_ARG:
1861 if (!frame
1862 || !frame->call
1863 || !frame->call->func
1864 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1865 break;
1866 }
1867 ADD_OP2_TRACE_GUARD();
1868 ADD_OP1_TRACE_GUARD();
1869 break;
1870 case ZEND_PRE_INC:
1871 case ZEND_PRE_DEC:
1872 case ZEND_POST_INC:
1873 case ZEND_POST_DEC:
1874 if (opline->op1_type != IS_CV) {
1875 break;
1876 }
1877 ADD_OP1_TRACE_GUARD();
1878 break;
1879 case ZEND_ASSIGN:
1880 if (opline->op1_type != IS_CV) {
1881 break;
1882 }
1883 ADD_OP2_TRACE_GUARD();
1884 if (op1_type != IS_UNKNOWN
1885 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1886 ADD_OP1_TRACE_GUARD();
1887 }
1888 break;
1889 case ZEND_CAST:
1890 if (opline->extended_value != op1_type) {
1891 break;
1892 }
1893 ADD_OP1_TRACE_GUARD();
1894 break;
1895 case ZEND_JMPZ:
1896 case ZEND_JMPNZ:
1897 case ZEND_JMPZ_EX:
1898 case ZEND_JMPNZ_EX:
1899 case ZEND_BOOL:
1900 case ZEND_BOOL_NOT:
1901 ADD_OP1_TRACE_GUARD();
1902 break;
1903 case ZEND_ISSET_ISEMPTY_CV:
1904 if ((opline->extended_value & ZEND_ISEMPTY)) {
1905 // TODO: support for empty() ???
1906 break;
1907 }
1908 ADD_OP1_TRACE_GUARD();
1909 break;
1910 case ZEND_IN_ARRAY:
1911 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
1912 break;
1913 }
1914 ADD_OP1_TRACE_GUARD();
1915 break;
1916 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
1917 if ((opline->extended_value & ZEND_ISEMPTY)) {
1918 // TODO: support for empty() ???
1919 break;
1920 }
1921 ZEND_FALLTHROUGH;
1922 case ZEND_FETCH_DIM_R:
1923 case ZEND_FETCH_DIM_IS:
1924 case ZEND_FETCH_LIST_R:
1925 ADD_OP1_TRACE_GUARD();
1926 ADD_OP2_TRACE_GUARD();
1927
1928 if (op1_type == IS_ARRAY
1929 && opline->op1_type != IS_CONST
1930 && ((opline->op2_type == IS_CONST
1931 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1932 || (opline->op2_type != IS_CONST
1933 && op2_type == IS_LONG))) {
1934
1935 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1936
1937 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1938 info->type |= MAY_BE_PACKED_GUARD;
1939 if (orig_op1_type & IS_TRACE_PACKED) {
1940 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1941 } else {
1942 info->type &= ~MAY_BE_ARRAY_PACKED;
1943 }
1944 }
1945 }
1946 break;
1947 case ZEND_FETCH_DIM_W:
1948 case ZEND_FETCH_DIM_RW:
1949 // case ZEND_FETCH_DIM_UNSET:
1950 case ZEND_FETCH_LIST_W:
1951 if (opline->op1_type != IS_CV
1952 && (orig_op1_type == IS_UNKNOWN
1953 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
1954 break;
1955 }
1956 ADD_OP1_TRACE_GUARD();
1957 ADD_OP2_TRACE_GUARD();
1958 if (op1_type == IS_ARRAY
1959 && !(orig_op1_type & IS_TRACE_PACKED)
1960 && ((opline->op2_type == IS_CONST
1961 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1962 || (opline->op2_type != IS_CONST
1963 && op2_type == IS_LONG))) {
1964
1965 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1966
1967 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1968 info->type |= MAY_BE_PACKED_GUARD;
1969 info->type &= ~MAY_BE_ARRAY_PACKED;
1970 }
1971 }
1972 break;
1973 case ZEND_SEND_VAL_EX:
1974 case ZEND_SEND_VAR_EX:
1975 case ZEND_SEND_VAR_NO_REF_EX:
1976 if (opline->op2_type == IS_CONST) {
1977 /* Named parameters not supported in JIT */
1978 break;
1979 }
1980 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
1981 goto propagate_arg;
1982 }
1983 ZEND_FALLTHROUGH;
1984 case ZEND_SEND_VAL:
1985 case ZEND_SEND_VAR:
1986 case ZEND_SEND_VAR_NO_REF:
1987 case ZEND_SEND_FUNC_ARG:
1988 if (opline->op2_type == IS_CONST) {
1989 /* Named parameters not supported in JIT */
1990 break;
1991 }
1992 ADD_OP1_TRACE_GUARD();
1993 propagate_arg:
1994 /* Propagate argument type */
1995 if (frame->call
1996 && frame->call->func
1997 && frame->call->func->type == ZEND_USER_FUNCTION
1998 && opline->op2.num <= frame->call->func->op_array.num_args) {
1999 uint32_t info;
2000
2001 if (opline->op1_type == IS_CONST) {
2002 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
2003 } else {
2004 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2005 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
2006 }
2007 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
2008 zend_arg_info *arg_info;
2009
2010 ZEND_ASSERT(frame->call->func->op_array.arg_info);
2011 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
2012 if (ZEND_TYPE_IS_SET(arg_info->type)) {
2013 zend_class_entry *ce;
2014 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2015 info &= tmp;
2016 if (!info) {
2017 break;
2018 }
2019 }
2020 }
2021 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2022 info |= MAY_BE_RCN;
2023 }
2024 if (info & MAY_BE_UNDEF) {
2025 info |= MAY_BE_NULL;
2026 info &= ~MAY_BE_UNDEF;
2027 }
2028 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2029 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2030 }
2031 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2032 }
2033 break;
2034 case ZEND_RETURN:
2035 ADD_OP1_TRACE_GUARD();
2036 /* Propagate return value types */
2037 if (opline->op1_type == IS_UNUSED) {
2038 return_value_info.type = MAY_BE_NULL;
2039 } else if (opline->op1_type == IS_CONST) {
2040 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2041 } else {
2042 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2043 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2044 if (return_value_info.type & MAY_BE_UNDEF) {
2045 return_value_info.type &= ~MAY_BE_UNDEF;
2046 return_value_info.type |= MAY_BE_NULL;
2047 }
2048 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2049 /* CVs are going to be destructed and the reference-counter
2050 of return value may be decremented to 1 */
2051 return_value_info.type |= MAY_BE_RC1;
2052 }
2053 return_value_info.type &= ~MAY_BE_GUARD;
2054 }
2055 break;
2056 case ZEND_CHECK_FUNC_ARG:
2057 if (!frame
2058 || !frame->call
2059 || !frame->call->func) {
2060 break;
2061 }
2062 if (opline->op2_type == IS_CONST
2063 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2064 /* Named parameters not supported in JIT */
2065 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2066 break;
2067 }
2068 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2069 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2070 } else {
2071 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2072 }
2073 break;
2074 case ZEND_FETCH_OBJ_FUNC_ARG:
2075 if (!frame
2076 || !frame->call
2077 || !frame->call->func
2078 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2079 break;
2080 }
2081 ZEND_FALLTHROUGH;
2082 case ZEND_FETCH_OBJ_R:
2083 case ZEND_FETCH_OBJ_IS:
2084 case ZEND_FETCH_OBJ_W:
2085 if (opline->op2_type != IS_CONST
2086 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2087 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2088 break;
2089 }
2090 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2091 ADD_OP1_TRACE_GUARD();
2092 }
2093 break;
2094 case ZEND_INIT_METHOD_CALL:
2095 if (opline->op2_type != IS_CONST
2096 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2097 break;
2098 }
2099 ADD_OP1_TRACE_GUARD();
2100 break;
2101 case ZEND_INIT_DYNAMIC_CALL:
2102 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2103 ADD_OP2_TRACE_GUARD();
2104 }
2105 break;
2106 case ZEND_SEND_ARRAY:
2107 case ZEND_SEND_UNPACK:
2108 case ZEND_CHECK_UNDEF_ARGS:
2109 case ZEND_INCLUDE_OR_EVAL:
2110 max_used_stack = used_stack = -1;
2111 break;
2112 case ZEND_TYPE_CHECK:
2113 if (opline->extended_value == MAY_BE_RESOURCE) {
2114 // TODO: support for is_resource() ???
2115 break;
2116 }
2117 if (op1_type != IS_UNKNOWN
2118 && (opline->extended_value == (1 << op1_type)
2119 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2120 /* add guards only for exact checks, to avoid code duplication */
2121 ADD_OP1_TRACE_GUARD();
2122 }
2123 break;
2124 case ZEND_ROPE_INIT:
2125 case ZEND_ROPE_ADD:
2126 case ZEND_ROPE_END:
2127 if (op2_type == IS_STRING) {
2128 ADD_OP2_TRACE_GUARD();
2129 }
2130 break;
2131 default:
2132 break;
2133 }
2134 len = zend_jit_trace_op_len(opline);
2135 if (ssa->var_info) {
2136 /* Add statically inferred ranges */
2137 if (ssa_ops[idx].op1_def >= 0) {
2138 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2139 }
2140 if (ssa_ops[idx].op2_def >= 0) {
2141 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2142 }
2143 if (ssa_ops[idx].result_def >= 0) {
2144 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2145 }
2146 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2147 if (ssa_ops[idx+1].op1_def >= 0) {
2148 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2149 }
2150 if (ssa_ops[idx+1].op2_def >= 0) {
2151 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2152 }
2153 if (ssa_ops[idx+1].result_def >= 0) {
2154 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2155 }
2156 }
2157 } else {
2158 if (ssa_ops[idx].op1_def >= 0) {
2159 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2160 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2161 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2162 }
2163 }
2164 if (ssa_ops[idx].op2_def >= 0) {
2165 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2166 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2167 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2168 }
2169 }
2170 if (ssa_ops[idx].result_def >= 0) {
2171 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2172 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2173 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2174 }
2175 }
2176 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2177 if (ssa_ops[idx+1].op1_def >= 0) {
2178 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2179 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2180 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2181 }
2182 }
2183 if (ssa_ops[idx+1].op2_def >= 0) {
2184 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2185 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2186 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2187 }
2188 }
2189 if (ssa_ops[idx+1].result_def >= 0) {
2190 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2191 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2192 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2193 }
2194 }
2195 }
2196 }
2197 if (opline->opcode == ZEND_RECV_INIT
2198 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2199 /* RECV_INIT always copy the constant */
2200 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2201 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2202 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2203 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2204 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2205 }
2206 } else {
2207 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2208 // TODO:
2209 assert(0);
2210 }
2211 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2212 && ssa_ops[idx].op1_def >= 0
2213 && op1_type == IS_ARRAY
2214 && (orig_op1_type & IS_TRACE_PACKED)
2215 && val_type != IS_UNKNOWN
2216 && val_type != IS_UNDEF
2217 && ((opline->op2_type == IS_CONST
2218 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2219 || (opline->op2_type != IS_CONST
2220 && op2_type == IS_LONG))) {
2221 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2222
2223 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2224 }
2225 }
2226 if (ssa->var_info) {
2227 /* Add statically inferred restrictions */
2228 if (ssa_ops[idx].op1_def >= 0) {
2229 if (opline->opcode == ZEND_SEND_VAR_EX
2230 && frame
2231 && frame->call
2232 && frame->call->func
2233 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2234 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2235 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2236 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2237 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2238 }
2239 } else {
2240 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2241 }
2242 }
2243 if (ssa_ops[idx].op2_def >= 0) {
2244 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2245 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2246 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2247 }
2248 }
2249 if (ssa_ops[idx].result_def >= 0) {
2250 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2251 }
2252 }
2253 idx++;
2254 while (len > 1) {
2255 opline++;
2256 if (opline->opcode != ZEND_OP_DATA) {
2257 if (ssa->var_info) {
2258 /* Add statically inferred ranges */
2259 if (ssa_ops[idx].op1_def >= 0) {
2260 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2261 }
2262 if (ssa_ops[idx].op2_def >= 0) {
2263 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2264 }
2265 if (ssa_ops[idx].result_def >= 0) {
2266 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2267 }
2268 } else {
2269 if (ssa_ops[idx].op1_def >= 0) {
2270 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2271 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2272 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2273 }
2274 }
2275 if (ssa_ops[idx].op2_def >= 0) {
2276 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2277 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2278 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2279 }
2280 }
2281 if (ssa_ops[idx].result_def >= 0) {
2282 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2283 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2284 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2285 }
2286 }
2287 }
2288 if (opline->opcode == ZEND_RECV_INIT
2289 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2290 /* RECV_INIT always copy the constant */
2291 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2292 } else {
2293 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2294 // TODO:
2295 assert(0);
2296 }
2297 }
2298 }
2299 if (ssa->var_info) {
2300 /* Add statically inferred restrictions */
2301 if (ssa_ops[idx].op1_def >= 0) {
2302 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2303 }
2304 if (ssa_ops[idx].op2_def >= 0) {
2305 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2306 }
2307 if (ssa_ops[idx].result_def >= 0) {
2308 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2309 }
2310 }
2311 idx++;
2312 len--;
2313 }
2314
2315 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2316 op_array = p->op_array;
2317 jit_extension =
2318 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2319 ssa = &jit_extension->func_info.ssa;
2320
2321 call = frame->call;
2322 if (!call) {
2323 /* Trace missed INIT_FCALL opcode */
2324 call = top;
2325 TRACE_FRAME_INIT(call, op_array, 0, 0);
2326 call->used_stack = 0;
2327 top = zend_jit_trace_call_frame(top, op_array);
2328 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2329 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
2330 }
2331 } else {
2332 ZEND_ASSERT(&call->func->op_array == op_array);
2333 }
2334 frame->call = call->prev;
2335 call->prev = frame;
2336 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2337 frame = call;
2338
2339 level++;
2340 i = 0;
2341 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2342 while (i < op_array->last_var) {
2343 ssa_vars[v].var = i;
2344 if (i < op_array->num_args) {
2345 if (ssa->var_info
2346 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2347 /* pass */
2348 } else {
2349 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2350 if (op_array->arg_info) {
2351 zend_arg_info *arg_info = &op_array->arg_info[i];
2352 zend_class_entry *ce;
2353 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2354
2355 if (ZEND_ARG_SEND_MODE(arg_info)) {
2356 tmp |= MAY_BE_REF;
2357 }
2358 ssa_var_info[v].type = tmp;
2359 ssa_var_info[v].ce = ce;
2360 ssa_var_info[v].is_instanceof = 1;
2361 } else {
2362 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2363 }
2364 }
2365 } else {
2366 if (ssa->vars) {
2367 ssa_vars[v].no_val = ssa->vars[i].no_val;
2368 ssa_vars[v].alias = ssa->vars[i].alias;
2369 } else {
2370 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2371 }
2372 if (ssa_vars[v].alias == NO_ALIAS) {
2373 ssa_var_info[v].type = MAY_BE_UNDEF;
2374 } else {
2375 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2376 }
2377 }
2378 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2379 && i < op_array->num_args) {
2380 /* Propagate argument type */
2381 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2382 }
2383 i++;
2384 v++;
2385 }
2386 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2387 op_array = p->op_array;
2388 jit_extension =
2389 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2390 ssa = &jit_extension->func_info.ssa;
2391 if (level == 0) {
2392 i = 0;
2393 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2394 while (i < op_array->last_var) {
2395 ssa_vars[v].var = i;
2396 if (!ssa->var_info
2397 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2398 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2399 }
2400 i++;
2401 v++;
2402 }
2403 while (i < op_array->last_var + op_array->T) {
2404 ssa_vars[v].var = i;
2405 if (!ssa->var_info
2406 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2407 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2408 }
2409 i++;
2410 v++;
2411 }
2412 if (return_value_info.type != 0) {
2413 zend_jit_trace_rec *q = p + 1;
2414 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2415 q++;
2416 }
2417 if (q->op == ZEND_JIT_TRACE_VM
2418 || (q->op == ZEND_JIT_TRACE_END
2419 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2420 const zend_op *opline = q->opline - 1;
2421 if (opline->result_type != IS_UNUSED) {
2422 ssa_var_info[
2423 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2424 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2425 }
2426 }
2427 memset(&return_value_info, 0, sizeof(return_value_info));
2428 }
2429 } else {
2430 level--;
2431 if (return_value_info.type != 0) {
2432 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2433 const zend_op *opline = (p+1)->opline - 1;
2434 if (opline->result_type != IS_UNUSED) {
2435 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2436 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2437 }
2438 }
2439 }
2440 memset(&return_value_info, 0, sizeof(return_value_info));
2441 }
2442 }
2443
2444 top = frame;
2445 if (frame->prev) {
2446 if (used_stack > 0) {
2447 used_stack -= frame->used_stack;
2448 }
2449 frame = frame->prev;
2450 ZEND_ASSERT(&frame->func->op_array == op_array);
2451 } else {
2452 max_used_stack = used_stack = -1;
2453 frame = zend_jit_trace_ret_frame(frame, op_array);
2454 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2455 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2456 frame->used_stack = 0;
2457 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2458 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
2459 }
2460 }
2461
2462 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2463 call = top;
2464 TRACE_FRAME_INIT(call, p->func, 0, 0);
2465 call->prev = frame->call;
2466 call->used_stack = 0;
2467 frame->call = call;
2468 top = zend_jit_trace_call_frame(top, p->op_array);
2469 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2470 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2471 SET_STACK_INFO(call->stack, i, -1);
2472 }
2473 }
2474 if (used_stack >= 0
2475 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2476 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2477 max_used_stack = used_stack = -1;
2478 } else {
2479 const zend_op *opline = (p-1)->opline;
2480
2481 switch (opline->opcode) {
2482 case ZEND_INIT_FCALL:
2483 case ZEND_INIT_FCALL_BY_NAME:
2484 case ZEND_INIT_NS_FCALL_BY_NAME:
2485 case ZEND_INIT_METHOD_CALL:
2486 case ZEND_INIT_DYNAMIC_CALL:
2487 //case ZEND_INIT_STATIC_METHOD_CALL:
2488 //case ZEND_INIT_USER_CALL:
2489 //case ZEND_NEW:
2490 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2491 used_stack += frame->used_stack;
2492 if (used_stack > max_used_stack) {
2493 max_used_stack = used_stack;
2494 }
2495 break;
2496 default:
2497 max_used_stack = used_stack = -1;
2498 }
2499 }
2500 }
2501 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2502 call = frame->call;
2503 if (call) {
2504 top = call;
2505 frame->call = call->prev;
2506 }
2507
2508 if (idx > 0
2509 && ssa_ops[idx-1].result_def >= 0
2510 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2511 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2512 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2513 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2514 opline->opcode == ZEND_DO_FCALL ||
2515 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2516
2517 if (opline->result_type != IS_UNDEF) {
2518 zend_class_entry *ce;
2519 const zend_function *func = p->func;
2520 zend_arg_info *ret_info = func->common.arg_info - 1;
2521 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2522
2523 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2524 }
2525 }
2526 } else if (p->op == ZEND_JIT_TRACE_END) {
2527 break;
2528 }
2529 }
2530
2531 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2532
2533 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2534 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2535 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2536 /* Propagate guards through Phi sources */
2537 zend_ssa_phi *phi = tssa->blocks[1].phis;
2538
2539 op_array = trace_buffer->op_array;
2540 jit_extension =
2541 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2542 ssa = &jit_extension->func_info.ssa;
2543
2544 while (phi) {
2545 uint32_t t = ssa_var_info[phi->ssa_var].type;
2546
2547 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2548 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2549 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2550
2551 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2552 if (!((t0 | t1) & MAY_BE_GUARD)) {
2553 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2554 }
2555 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2556 if (!(t1 & MAY_BE_GUARD)
2557 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2558 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2559 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2560 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2561 MAY_BE_GUARD;
2562 if (!(t0 & MAY_BE_ARRAY)) {
2563 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2564 }
2565 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2566 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2567 }
2568 ssa_var_info[phi->sources[0]].type = t0;
2569 ssa_var_info[phi->sources[0]].type = t0;
2570 }
2571 } else {
2572 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2573 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2574 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2575 MAY_BE_GUARD;
2576 if (!(t0 & MAY_BE_ARRAY)) {
2577 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2578 }
2579 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2580 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2581 }
2582 ssa_var_info[phi->sources[0]].type = t0;
2583 }
2584 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2585 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2586 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2587 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2588 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2589 MAY_BE_GUARD;
2590 if (!(t1 & MAY_BE_ARRAY)) {
2591 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2592 }
2593 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2594 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2595 }
2596 ssa_var_info[phi->sources[1]].type = t1;
2597 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2598 }
2599 }
2600 }
2601 t = ssa_var_info[phi->ssa_var].type;
2602 }
2603
2604 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2605 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2606 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2607
2608 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2609 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2610 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2611 }
2612 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2613 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2614 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2615 ssa_var_info[phi->sources[0]].type =
2616 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2617 }
2618 }
2619 }
2620 phi = phi->next;
2621 }
2622 }
2623
2624 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2625 if (parent_trace) {
2626 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2627 ZEND_JIT_TRACE_NUM,
2628 parent_trace,
2629 exit_num,
2630 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2631 trace_buffer->op_array->scope ? "::" : "",
2632 trace_buffer->op_array->function_name ?
2633 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2634 ZSTR_VAL(trace_buffer->op_array->filename),
2635 trace_buffer[1].opline->lineno);
2636 } else {
2637 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2638 ZEND_JIT_TRACE_NUM,
2639 zend_jit_trace_star_desc(trace_buffer->start),
2640 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2641 trace_buffer->op_array->scope ? "::" : "",
2642 trace_buffer->op_array->function_name ?
2643 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2644 ZSTR_VAL(trace_buffer->op_array->filename),
2645 trace_buffer[1].opline->lineno);
2646 }
2647 zend_jit_dump_trace(trace_buffer, tssa);
2648 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2649 uint32_t idx = trace_buffer[1].last;
2650 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2651 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2652 ZEND_JIT_TRACE_NUM,
2653 link_to);
2654 } else {
2655 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2656 ZEND_JIT_TRACE_NUM,
2657 zend_jit_trace_stop_description[trace_buffer->stop]);
2658 }
2659 }
2660
2661 return tssa;
2662 }
2663
zend_jit_close_var(zend_jit_trace_stack * stack,uint32_t n,int * start,int * end,uint8_t * flags,int line)2664 static void zend_jit_close_var(zend_jit_trace_stack *stack, uint32_t n, int *start, int *end, uint8_t *flags, int line)
2665 {
2666 int32_t var = STACK_VAR(stack, n);
2667
2668 if (var >= 0 && start[var] >= 0 && !(flags[var] & ZREG_LAST_USE)) {
2669 // TODO: shrink interval to last side exit ????
2670 end[var] = line;
2671 }
2672 }
2673
zend_jit_trace_use_var(int line,int var,int def,int use_chain,int * start,int * end,uint8_t * flags,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2674 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, int *start, int *end, uint8_t *flags, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2675 {
2676 ZEND_ASSERT(start[var] >= 0);
2677 ZEND_ASSERT(!(flags[var] & ZREG_LAST_USE));
2678 end[var] = line;
2679 if (def >= 0) {
2680 flags[var] |= ZREG_LAST_USE;
2681 } else if (use_chain < 0 && (flags[var] & (ZREG_LOAD|ZREG_STORE))) {
2682 flags[var] |= ZREG_LAST_USE;
2683 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2684 /* pass */
2685 } else if (op_array_ssa->vars) {
2686 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2687
2688 if (ssa->ops[line].op1_use == var) {
2689 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2690 flags[var] |= ZREG_LAST_USE;
2691 }
2692 } else if (ssa->ops[line].op2_use == var) {
2693 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2694 flags[var] |= ZREG_LAST_USE;
2695 }
2696 } else if (ssa->ops[line].result_use == var) {
2697 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2698 flags[var] |= ZREG_LAST_USE;
2699 }
2700 }
2701 }
2702 }
2703
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2704 static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2705 {
2706 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2707 zend_jit_trace_rec *p;
2708 const zend_op_array *op_array;
2709 zend_jit_op_array_trace_extension *jit_extension;
2710 const zend_ssa *op_array_ssa;
2711 const zend_ssa_op *ssa_op;
2712 int i, j, idx, count, level;
2713 int last_idx = -1;
2714 int *start, *end;
2715 uint8_t *flags;
2716 const zend_op_array **vars_op_array;
2717 zend_lifetime_interval **intervals, *list, *ival;
2718 void *checkpoint;
2719 zend_jit_trace_stack_frame *frame;
2720 zend_jit_trace_stack *stack;
2721 uint32_t parent_vars_count = parent_trace ?
2722 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2723 zend_jit_trace_stack *parent_stack = parent_trace ?
2724 zend_jit_traces[parent_trace].stack_map +
2725 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2726 ALLOCA_FLAG(use_heap);
2727
2728 ZEND_ASSERT(ssa->var_info != NULL);
2729
2730 start = do_alloca(sizeof(int) * ssa->vars_count * 2 +
2731 ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count),
2732 use_heap);
2733 if (!start) {
2734 return NULL;
2735 }
2736 end = start + ssa->vars_count;
2737 flags = (uint8_t*)(end + ssa->vars_count);
2738 checkpoint = zend_arena_checkpoint(CG(arena));
2739 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2740
2741 memset(start, -1, sizeof(int) * ssa->vars_count * 2);
2742 memset(flags, 0, sizeof(uint8_t) * ssa->vars_count);
2743 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2744
2745 op_array = trace_buffer->op_array;
2746 jit_extension =
2747 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2748 op_array_ssa = &jit_extension->func_info.ssa;
2749 frame = JIT_G(current_frame);
2750 frame->prev = NULL;
2751 frame->func = (const zend_function*)op_array;
2752 stack = frame->stack;
2753
2754 count = 0;
2755
2756 i = 0;
2757 j = op_array->last_var;
2758 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2759 j += op_array->T;
2760 }
2761 while (i < j) {
2762 SET_STACK_VAR(stack, i, i);
2763 vars_op_array[i] = op_array;
2764 /* We don't start intervals for variables used in Phi */
2765 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2766 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2767 && ssa->vars[i].alias == NO_ALIAS
2768 && zend_jit_var_supports_reg(ssa, i)) {
2769 start[i] = 0;
2770 if (i < parent_vars_count
2771 && STACK_REG(parent_stack, i) != ZREG_NONE
2772 && STACK_REG(parent_stack, i) < ZREG_NUM) {
2773 /* We will try to reuse register from parent trace */
2774 flags[i] = STACK_FLAGS(parent_stack, i);
2775 count += 2;
2776 } else {
2777 flags[i] = ZREG_LOAD;
2778 count++;
2779 }
2780 }
2781 i++;
2782 }
2783
2784 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2785 j = op_array->last_var + op_array->T;
2786 while (i < j) {
2787 SET_STACK_VAR(stack, i, -1);
2788 i++;
2789 }
2790 }
2791
2792 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2793 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2794 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2795 zend_ssa_phi *phi = ssa->blocks[1].phis;
2796
2797 while (phi) {
2798 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2799 vars_op_array[phi->ssa_var] = op_array;
2800 if (ssa->vars[phi->ssa_var].use_chain >= 0
2801 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2802 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2803 start[phi->ssa_var] = 0;
2804 count++;
2805 }
2806 phi = phi->next;
2807 }
2808 }
2809
2810 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2811 level = 0;
2812 ssa_op = ssa->ops;
2813 idx = 0;
2814 for (;;p++) {
2815 if (p->op == ZEND_JIT_TRACE_VM) {
2816 const zend_op *opline = p->opline;
2817 int len;
2818 bool support_opline;
2819
2820 support_opline =
2821 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2822
2823 if (support_opline
2824 && opline->opcode == ZEND_ASSIGN
2825 && opline->op1_type == IS_CV
2826 && ssa_op->op1_def >= 0
2827 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2828 /* avoid register allocation in case of possibility of indirect modification*/
2829 support_opline = 0;
2830 }
2831
2832 if (ssa_op->op1_use >= 0
2833 && start[ssa_op->op1_use] >= 0
2834 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2835 if (support_opline) {
2836 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2837 if (opline->op1_type != IS_CV) {
2838 if (opline->opcode == ZEND_CASE
2839 || opline->opcode == ZEND_CASE_STRICT
2840 || opline->opcode == ZEND_SWITCH_LONG
2841 || opline->opcode == ZEND_MATCH
2842 || opline->opcode == ZEND_FETCH_LIST_R
2843 || opline->opcode == ZEND_COPY_TMP
2844 || opline->opcode == ZEND_SWITCH_STRING
2845 || opline->opcode == ZEND_FE_FETCH_R
2846 || opline->opcode == ZEND_FE_FETCH_RW
2847 || opline->opcode == ZEND_FETCH_LIST_W
2848 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2849 || opline->opcode == ZEND_BIND_LEXICAL
2850 || opline->opcode == ZEND_ROPE_ADD) {
2851 /* The value is kept alive and may be used outside of the trace */
2852 flags[ssa_op->op1_use] |= ZREG_STORE;
2853 } else {
2854 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2855 }
2856 }
2857 } else {
2858 start[ssa_op->op1_use] = -1;
2859 end[ssa_op->op1_use] = -1;
2860 count--;
2861 }
2862 }
2863 if (ssa_op->op2_use >= 0
2864 && ssa_op->op2_use != ssa_op->op1_use
2865 && start[ssa_op->op2_use] >= 0
2866 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2867 if (support_opline) {
2868 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2869 if (opline->op2_type != IS_CV) {
2870 flags[ssa_op->op2_use] |= ZREG_LAST_USE;
2871 }
2872 } else {
2873 start[ssa_op->op2_use] = -1;
2874 end[ssa_op->op2_use] = -1;
2875 count--;
2876 }
2877 }
2878 if (ssa_op->result_use >= 0
2879 && ssa_op->result_use != ssa_op->op1_use
2880 && ssa_op->result_use != ssa_op->op2_use
2881 && start[ssa_op->result_use] >= 0
2882 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2883 if (support_opline) {
2884 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2885 } else {
2886 start[ssa_op->result_use] = -1;
2887 end[ssa_op->result_use] = -1;
2888 count--;
2889 }
2890 }
2891
2892 if (ssa_op->op1_def >= 0) {
2893 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2894 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2895 }
2896 if (ssa_op->op2_def >= 0) {
2897 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op2.var), start, end, flags, idx);
2898 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
2899 }
2900 if (ssa_op->result_def >= 0) {
2901 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2902 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2903 }
2904
2905 if (support_opline) {
2906 if (ssa_op->result_def >= 0
2907 && (ssa->vars[ssa_op->result_def].use_chain >= 0
2908 || ssa->vars[ssa_op->result_def].phi_use_chain)
2909 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
2910 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
2911 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
2912 || opline->opcode == ZEND_PRE_INC
2913 || opline->opcode == ZEND_PRE_DEC
2914 || opline->opcode == ZEND_POST_INC
2915 || opline->opcode == ZEND_POST_DEC
2916 || opline->opcode == ZEND_ADD
2917 || opline->opcode == ZEND_SUB
2918 || opline->opcode == ZEND_MUL
2919 || opline->opcode == ZEND_FETCH_DIM_R
2920 || opline->opcode == ZEND_FETCH_CONSTANT) {
2921 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
2922 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
2923 start[ssa_op->result_def] = idx;
2924 vars_op_array[ssa_op->result_def] = op_array;
2925 count++;
2926 }
2927 }
2928 }
2929 if (ssa_op->op1_def >= 0
2930 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2931 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2932 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2933 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2934 start[ssa_op->op1_def] = idx;
2935 vars_op_array[ssa_op->op1_def] = op_array;
2936 count++;
2937 }
2938 if (ssa_op->op2_def >= 0
2939 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
2940 || ssa->vars[ssa_op->op2_def].phi_use_chain)
2941 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
2942 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)) {
2943 start[ssa_op->op2_def] = idx;
2944 vars_op_array[ssa_op->op2_def] = op_array;
2945 count++;
2946 }
2947 }
2948
2949 len = zend_jit_trace_op_len(opline);
2950 switch (opline->opcode) {
2951 case ZEND_ASSIGN_DIM:
2952 case ZEND_ASSIGN_OBJ:
2953 case ZEND_ASSIGN_STATIC_PROP:
2954 case ZEND_ASSIGN_DIM_OP:
2955 case ZEND_ASSIGN_OBJ_OP:
2956 case ZEND_ASSIGN_STATIC_PROP_OP:
2957 case ZEND_ASSIGN_OBJ_REF:
2958 case ZEND_ASSIGN_STATIC_PROP_REF:
2959 /* OP_DATA */
2960 ssa_op++;
2961 opline++;
2962 if (ssa_op->op1_use >= 0
2963 && start[ssa_op->op1_use] >= 0
2964 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2965 if (support_opline) {
2966 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2967 if (opline->op1_type != IS_CV) {
2968 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2969 }
2970 } else {
2971 start[ssa_op->op1_use] = -1;
2972 end[ssa_op->op1_use] = -1;
2973 count--;
2974 }
2975 }
2976 if (ssa_op->op1_def >= 0) {
2977 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2978 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2979 if (support_opline
2980 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2981 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2982 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2983 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2984 start[ssa_op->op1_def] = idx;
2985 vars_op_array[ssa_op->op1_def] = op_array;
2986 count++;
2987 }
2988 }
2989 ssa_op++;
2990 opline++;
2991 idx+=2;
2992 break;
2993 case ZEND_RECV_INIT:
2994 ssa_op++;
2995 opline++;
2996 idx++;
2997 while (opline->opcode == ZEND_RECV_INIT) {
2998 /* RECV_INIT doesn't support registers */
2999 if (ssa_op->result_def >= 0) {
3000 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
3001 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3002 }
3003 ssa_op++;
3004 opline++;
3005 idx++;
3006 }
3007 break;
3008 case ZEND_BIND_GLOBAL:
3009 ssa_op++;
3010 opline++;
3011 idx++;
3012 while (opline->opcode == ZEND_BIND_GLOBAL) {
3013 /* BIND_GLOBAL doesn't support registers */
3014 if (ssa_op->op1_def >= 0) {
3015 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
3016 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3017 }
3018 ssa_op++;
3019 opline++;
3020 idx++;
3021 }
3022 break;
3023 default:
3024 ssa_op += len;
3025 idx += len;
3026 break;
3027 }
3028 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3029 /* New call frames */
3030 zend_jit_trace_stack_frame *prev_frame = frame;
3031
3032 frame = zend_jit_trace_call_frame(frame, op_array);
3033 frame->prev = prev_frame;
3034 frame->func = (const zend_function*)p->op_array;
3035 stack = frame->stack;
3036 op_array = p->op_array;
3037 jit_extension =
3038 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3039 op_array_ssa = &jit_extension->func_info.ssa;
3040 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3041 for (i = 0; i < op_array->last_var; i++) {
3042 SET_STACK_VAR(stack, i, j);
3043 vars_op_array[j] = op_array;
3044 if (ssa->vars[j].use_chain >= 0
3045 && ssa->vars[j].alias == NO_ALIAS
3046 && zend_jit_var_supports_reg(ssa, j)) {
3047 start[j] = idx;
3048 flags[j] = ZREG_LOAD;
3049 count++;
3050 }
3051 j++;
3052 }
3053 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3054 SET_STACK_VAR(stack, i, -1);
3055 }
3056 level++;
3057 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3058 /* Close exiting call frames */
3059 for (i = 0; i < op_array->last_var; i++) {
3060 zend_jit_close_var(stack, i, start, end, flags, idx-1);
3061 }
3062 op_array = p->op_array;
3063 jit_extension =
3064 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3065 op_array_ssa = &jit_extension->func_info.ssa;
3066 frame = zend_jit_trace_ret_frame(frame, op_array);
3067 stack = frame->stack;
3068 if (level == 0) {
3069 /* New return frames */
3070 frame->prev = NULL;
3071 frame->func = (const zend_function*)op_array;
3072 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3073 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3074 SET_STACK_VAR(stack, i, j);
3075 vars_op_array[j] = op_array;
3076 if (ssa->vars[j].use_chain >= 0
3077 && ssa->vars[j].alias == NO_ALIAS
3078 && zend_jit_var_supports_reg(ssa, j)) {
3079 start[j] = idx;
3080 flags[j] = ZREG_LOAD;
3081 count++;
3082 }
3083 j++;
3084 }
3085 } else {
3086 level--;
3087 }
3088 } else if (p->op == ZEND_JIT_TRACE_END) {
3089 break;
3090 }
3091 }
3092
3093 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3094 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3095 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3096 zend_ssa_phi *phi = ssa->blocks[1].phis;
3097
3098 while (phi) {
3099 i = phi->sources[1];
3100 if (start[i] >= 0 && !ssa->vars[phi->ssa_var].no_val) {
3101 end[i] = idx;
3102 flags[i] &= ~ZREG_LAST_USE;
3103 }
3104 phi = phi->next;
3105 }
3106
3107 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3108 for (i = 0; i < op_array->last_var; i++) {
3109 if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) {
3110 end[i] = idx;
3111 flags[i] &= ~ZREG_LAST_USE;
3112 } else {
3113 zend_jit_close_var(stack, i, start, end, flags, idx);
3114 }
3115 }
3116 }
3117 } else {
3118 last_idx = idx;
3119 for (i = 0; i < op_array->last_var; i++) {
3120 zend_jit_close_var(stack, i, start, end, flags, idx);
3121 }
3122 while (frame->prev) {
3123 frame = frame->prev;
3124 op_array = &frame->func->op_array;
3125 stack = frame->stack;
3126 for (i = 0; i < op_array->last_var; i++) {
3127 zend_jit_close_var(stack, i, start, end, flags, idx);
3128 }
3129 }
3130 }
3131
3132 if (!count) {
3133 free_alloca(start, use_heap);
3134 zend_arena_release(&CG(arena), checkpoint);
3135 return NULL;
3136 }
3137
3138 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval));
3139 memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count);
3140 list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count);
3141 j = 0;
3142 for (i = 0; i < ssa->vars_count; i++) {
3143 if (start[i] >= 0 && end[i] >= 0) {
3144 ZEND_ASSERT(j < count);
3145 if ((flags[i] & ZREG_LOAD) &&
3146 (flags[i] & ZREG_LAST_USE) &&
3147 end[i] == ssa->vars[i].use_chain) {
3148 /* skip life range with single use */
3149 continue;
3150 }
3151 intervals[i] = &list[j];
3152 list[j].ssa_var = i;
3153 list[j].reg = ZREG_NONE;
3154 list[j].flags = flags[i];
3155 list[j].range.start = start[i];
3156 list[j].range.end = end[i];
3157 list[j].range.next = NULL;
3158 list[j].hint = NULL;
3159 list[j].used_as_hint = NULL;
3160 list[j].list_next = NULL;
3161 j++;
3162 }
3163 }
3164 count = j;
3165 free_alloca(start, use_heap);
3166 start = end = NULL;
3167
3168 if (!count) {
3169 zend_arena_release(&CG(arena), checkpoint);
3170 return NULL;
3171 }
3172
3173 /* Add hints */
3174 if (parent_vars_count) {
3175 i = trace_buffer->op_array->last_var;
3176 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3177 i += trace_buffer->op_array->T;
3178 }
3179 if ((uint32_t)i > parent_vars_count) {
3180 i = parent_vars_count;
3181 }
3182 while (i > 0) {
3183 i--;
3184 if (intervals[i]
3185 && STACK_REG(parent_stack, i) != ZREG_NONE
3186 && STACK_REG(parent_stack, i) < ZREG_NUM) {
3187 list[j].ssa_var = - 1;
3188 list[j].reg = STACK_REG(parent_stack, i);
3189 list[j].flags = 0;
3190 list[j].range.start = -1;
3191 list[j].range.end = -1;
3192 list[j].range.next = NULL;
3193 list[j].hint = NULL;
3194 list[j].used_as_hint = NULL;
3195 list[j].list_next = NULL;
3196 intervals[i]->hint = &list[j];
3197 j++;
3198 }
3199 }
3200 }
3201
3202 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3203 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3204 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3205 zend_ssa_phi *phi = ssa->blocks[1].phis;
3206
3207 while (phi) {
3208 if (intervals[phi->ssa_var]) {
3209 if (intervals[phi->sources[1]]
3210 && (ssa->var_info[phi->sources[1]].type & MAY_BE_ANY) ==
3211 (ssa->var_info[phi->ssa_var].type & MAY_BE_ANY)) {
3212 intervals[phi->sources[1]]->hint = intervals[phi->ssa_var];
3213 }
3214 }
3215 phi = phi->next;
3216 }
3217 }
3218
3219 for (i = 0; i < ssa->vars_count; i++) {
3220 if (intervals[i] && !intervals[i]->hint) {
3221
3222 if (ssa->vars[i].definition >= 0) {
3223 uint32_t line = ssa->vars[i].definition;
3224 const zend_op *opline = ssa_opcodes[line];
3225
3226 switch (opline->opcode) {
3227 case ZEND_QM_ASSIGN:
3228 case ZEND_POST_INC:
3229 case ZEND_POST_DEC:
3230 if (ssa->ops[line].op1_use >= 0 &&
3231 intervals[ssa->ops[line].op1_use] &&
3232 (i == ssa->ops[line].op1_def ||
3233 (i == ssa->ops[line].result_def &&
3234 (ssa->ops[line].op1_def < 0 ||
3235 !intervals[ssa->ops[line].op1_def])))) {
3236 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3237 }
3238 break;
3239 case ZEND_SEND_VAR:
3240 if (opline->op2_type == IS_CONST) {
3241 /* Named parameters not supported in JIT */
3242 break;
3243 }
3244 case ZEND_PRE_INC:
3245 case ZEND_PRE_DEC:
3246 if (i == ssa->ops[line].op1_def &&
3247 ssa->ops[line].op1_use >= 0 &&
3248 intervals[ssa->ops[line].op1_use]) {
3249 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3250 }
3251 break;
3252 case ZEND_ASSIGN:
3253 if (ssa->ops[line].op2_use >= 0 &&
3254 intervals[ssa->ops[line].op2_use] &&
3255 (i == ssa->ops[line].op2_def ||
3256 (i == ssa->ops[line].op1_def &&
3257 (ssa->ops[line].op2_def < 0 ||
3258 !intervals[ssa->ops[line].op2_def])) ||
3259 (i == ssa->ops[line].result_def &&
3260 (ssa->ops[line].op2_def < 0 ||
3261 !intervals[ssa->ops[line].op2_def]) &&
3262 (ssa->ops[line].op1_def < 0 ||
3263 !intervals[ssa->ops[line].op1_def])))) {
3264 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3265 }
3266 break;
3267 case ZEND_SUB:
3268 case ZEND_ADD:
3269 case ZEND_MUL:
3270 case ZEND_BW_OR:
3271 case ZEND_BW_AND:
3272 case ZEND_BW_XOR:
3273 if (i == ssa->ops[line].result_def) {
3274 if (ssa->ops[line].op1_use >= 0 &&
3275 intervals[ssa->ops[line].op1_use] &&
3276 ssa->ops[line].op1_use_chain < 0 &&
3277 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
3278 (ssa->var_info[i].type & MAY_BE_ANY) ==
3279 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
3280
3281 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op1_use].definition_phi;
3282 if (phi &&
3283 intervals[phi->sources[1]] &&
3284 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op1_use]) {
3285 break;
3286 }
3287 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3288 } else if (opline->opcode != ZEND_SUB &&
3289 ssa->ops[line].op2_use >= 0 &&
3290 intervals[ssa->ops[line].op2_use] &&
3291 ssa->ops[line].op2_use_chain < 0 &&
3292 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
3293 (ssa->var_info[i].type & MAY_BE_ANY) ==
3294 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
3295
3296 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op2_use].definition_phi;
3297 if (phi &&
3298 intervals[phi->sources[1]] &&
3299 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op2_use]) {
3300 break;
3301 }
3302 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3303 }
3304 }
3305 break;
3306 }
3307 }
3308 }
3309 }
3310
3311 list = zend_jit_sort_intervals(intervals, ssa->vars_count);
3312
3313 if (list) {
3314 ival = list;
3315 while (ival) {
3316 if (ival->hint) {
3317 ival->hint->used_as_hint = ival;
3318 }
3319 ival = ival->list_next;
3320 }
3321 }
3322
3323 if (list) {
3324 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3325 fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM);
3326 ival = list;
3327 while (ival) {
3328 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3329 ival = ival->list_next;
3330 }
3331 }
3332 }
3333
3334 /* Linear Scan Register Allocation (op_array is not actually used, only fn_flags matters) */
3335 list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list);
3336
3337 if (list) {
3338 zend_lifetime_interval *ival, *next;
3339
3340 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
3341 ival = list;
3342 count = 0;
3343 while (ival != NULL) {
3344 ZEND_ASSERT(ival->reg != ZREG_NONE);
3345 count++;
3346 next = ival->list_next;
3347 ival->list_next = intervals[ival->ssa_var];
3348 intervals[ival->ssa_var] = ival;
3349 ival = next;
3350 }
3351
3352 if (!count) {
3353 zend_arena_release(&CG(arena), checkpoint);
3354 return NULL;
3355 }
3356
3357 /* Add LOAD flag to registers that can't reuse register from parent trace */
3358 if (parent_vars_count) {
3359 i = trace_buffer->op_array->last_var;
3360 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3361 i += trace_buffer->op_array->T;
3362 }
3363 if ((uint32_t)i > parent_vars_count) {
3364 i = parent_vars_count;
3365 }
3366 while (i > 0) {
3367 i--;
3368 if (intervals[i] && intervals[i]->reg != STACK_REG(parent_stack, i)) {
3369 intervals[i]->flags |= ZREG_LOAD;
3370 }
3371 }
3372 }
3373
3374 /* SSA resolution */
3375 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3376 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3377 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3378 zend_ssa_phi *phi = ssa->blocks[1].phis;
3379
3380 while (phi) {
3381 int def = phi->ssa_var;
3382 int use = phi->sources[1];
3383
3384 if (intervals[def]) {
3385 if (!intervals[use]) {
3386 intervals[def]->flags |= ZREG_LOAD;
3387 if ((intervals[def]->flags & ZREG_LAST_USE)
3388 && ssa->vars[def].use_chain >= 0
3389 && ssa->vars[def].use_chain == intervals[def]->range.end) {
3390 /* remove interval used once */
3391 intervals[def] = NULL;
3392 count--;
3393 }
3394 } else if (intervals[def]->reg != intervals[use]->reg) {
3395 intervals[def]->flags |= ZREG_LOAD;
3396 if (ssa->vars[use].use_chain >= 0) {
3397 intervals[use]->flags |= ZREG_STORE;
3398 } else {
3399 intervals[use] = NULL;
3400 count--;
3401 }
3402 } else {
3403 use = phi->sources[0];
3404 ZEND_ASSERT(!intervals[use]);
3405 intervals[use] = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
3406 intervals[use]->ssa_var = phi->sources[0];
3407 intervals[use]->reg = intervals[def]->reg;
3408 intervals[use]->flags = ZREG_LOAD;
3409 intervals[use]->range.start = 0;
3410 intervals[use]->range.end = 0;
3411 intervals[use]->range.next = NULL;
3412 intervals[use]->hint = NULL;
3413 intervals[use]->used_as_hint = NULL;
3414 intervals[use]->list_next = NULL;
3415 }
3416 } else if (intervals[use]
3417 && (!ssa->vars[def].no_val
3418 || ssa->var_info[def].type != ssa->var_info[use].type)) {
3419 if (ssa->vars[use].use_chain >= 0) {
3420 intervals[use]->flags |= ZREG_STORE;
3421 } else {
3422 intervals[use] = NULL;
3423 count--;
3424 }
3425 }
3426 phi = phi->next;
3427 }
3428 } else {
3429 for (i = 0; i < ssa->vars_count; i++) {
3430 if (intervals[i]
3431 && intervals[i]->range.end == last_idx
3432 && !(intervals[i]->flags & (ZREG_LOAD|ZREG_STORE))) {
3433 intervals[i]->flags |= ZREG_STORE;
3434 }
3435 }
3436 }
3437
3438 if (!count) {
3439 zend_arena_release(&CG(arena), checkpoint);
3440 return NULL;
3441 }
3442
3443 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3444 fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM);
3445 for (i = 0; i < ssa->vars_count; i++) {
3446 ival = intervals[i];
3447 while (ival) {
3448 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3449 ival = ival->list_next;
3450 }
3451 }
3452 }
3453
3454 return intervals;
3455 }
3456
3457 zend_arena_release(&CG(arena), checkpoint); //???
3458 return NULL;
3459 }
3460
zend_jit_trace_clenup_stack(zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,zend_lifetime_interval ** ra)3461 static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, zend_lifetime_interval **ra)
3462 {
3463 uint32_t line = ssa_op - ssa->ops;
3464
3465 if (ssa_op->op1_use >= 0
3466 && ra[ssa_op->op1_use]
3467 && ra[ssa_op->op1_use]->range.end == line) {
3468 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
3469 }
3470 if (ssa_op->op2_use >= 0
3471 && ra[ssa_op->op2_use]
3472 && ra[ssa_op->op2_use]->range.end == line) {
3473 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op2.var), ZREG_NONE);
3474 }
3475 if (ssa_op->result_use >= 0
3476 && ra[ssa_op->result_use]
3477 && ra[ssa_op->result_use]->range.end == line) {
3478 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE);
3479 }
3480 }
3481
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3482 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3483 {
3484 zend_op *next_opline = (zend_op*)(opline + 1);
3485
3486 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3487 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3488 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3489 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3490 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3491 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3492 }
3493 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3494 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3495 }
3496 }
3497
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3498 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3499 {
3500 int var = ssa_op->result_def;
3501 int i;
3502 int use = ssa->vars[var].use_chain;
3503 const zend_op *opline;
3504
3505 if (use < 0
3506 || ssa->vars[var].phi_use_chain
3507 || ssa->ops[use].op1_use != var
3508 || ssa->ops[use].op1_use_chain != -1) {
3509 return 0;
3510 }
3511
3512 opline = ssa_opcodes[use];
3513 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3514 return (opline->op2_type == IS_CONST &&
3515 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3516 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3517 if (!JIT_G(current_frame)
3518 || !JIT_G(current_frame)->call
3519 || !JIT_G(current_frame)->call->func
3520 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3521 return 0;
3522 }
3523 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3524 && opline->opcode != ZEND_FETCH_OBJ_IS
3525 && opline->opcode != ZEND_FETCH_OBJ_W
3526 && opline->opcode != ZEND_ASSIGN_OBJ
3527 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3528 && opline->opcode != ZEND_PRE_INC_OBJ
3529 && opline->opcode != ZEND_PRE_DEC_OBJ
3530 && opline->opcode != ZEND_POST_INC_OBJ
3531 && opline->opcode != ZEND_POST_DEC_OBJ) {
3532 return 0;
3533 }
3534
3535 if (opline->op2_type != IS_CONST
3536 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3537 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3538 return 0;
3539 }
3540
3541 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3542 if (opline->op1_type == IS_CV
3543 && (opline+1)->op1_type == IS_CV
3544 && (opline+1)->op1.var == opline->op1.var) {
3545 /* skip $a->prop += $a; */
3546 return 0;
3547 }
3548 if (!zend_jit_supported_binary_op(
3549 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3550 return 0;
3551 }
3552 }
3553
3554 for (i = ssa->vars[var].definition; i < use; i++) {
3555 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3556 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3557 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3558 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3559 return 0;
3560 }
3561 }
3562
3563 return 1;
3564 }
3565
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3566 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3567 {
3568 uint32_t i;
3569
3570 for (i = 0; i < stack_size; i++) {
3571 if (STACK_REG(stack, i) != ZREG_NONE
3572 && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
3573 return 1;
3574 }
3575 }
3576 return 0;
3577 }
3578
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3579 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3580 {
3581 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3582 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3583 uint32_t stack_size;
3584 zend_jit_trace_stack *stack;
3585
3586 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3587 return 1;
3588 }
3589
3590 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3591 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3592 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3593 }
3594
zend_jit_trace_deoptimization(dasm_State ** Dst,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_lifetime_interval ** ra,bool polymorphic_side_trace)3595 static int zend_jit_trace_deoptimization(dasm_State **Dst,
3596 uint32_t flags,
3597 const zend_op *opline,
3598 zend_jit_trace_stack *parent_stack,
3599 int parent_vars_count,
3600 zend_ssa *ssa,
3601 zend_jit_trace_stack *stack,
3602 zend_lifetime_interval **ra,
3603 bool polymorphic_side_trace)
3604 {
3605 int i;
3606 bool has_constants = 0;
3607 bool has_unsaved_vars = 0;
3608
3609 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3610 for (i = 0; i < parent_vars_count; i++) {
3611 int8_t reg = STACK_REG(parent_stack, i);
3612
3613 if (reg != ZREG_NONE) {
3614 if (reg < ZREG_NUM) {
3615 if (ssa && ssa->vars[i].no_val) {
3616 /* pass */
3617 } else if (ra && ra[i] && ra[i]->reg == reg) {
3618 /* register already loaded by parent trace */
3619 if (stack) {
3620 SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i));
3621 }
3622 has_unsaved_vars = 1;
3623 } else {
3624 uint8_t type = STACK_TYPE(parent_stack, i);
3625
3626 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3627 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3628 STACK_MEM_TYPE(parent_stack, i) != type)) {
3629 return 0;
3630 }
3631 if (stack) {
3632 SET_STACK_TYPE(stack, i, type, 1);
3633 }
3634 }
3635 } else {
3636 /* delay custom deoptimization instructions to prevent register clobbering */
3637 has_constants = 1;
3638 }
3639 }
3640 }
3641
3642 if (has_unsaved_vars
3643 && (has_constants
3644 || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) {
3645 for (i = 0; i < parent_vars_count; i++) {
3646 int8_t reg = STACK_REG(parent_stack, i);
3647
3648 if (reg != ZREG_NONE) {
3649 if (reg < ZREG_NUM) {
3650 if (ssa && ssa->vars[i].no_val) {
3651 /* pass */
3652 } else if (ra && ra[i] && ra[i]->reg == reg) {
3653 uint8_t type = STACK_TYPE(parent_stack, i);
3654
3655 if (stack) {
3656 SET_STACK_TYPE(stack, i, type, 1);
3657 }
3658 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3659 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3660 STACK_MEM_TYPE(parent_stack, i) != type)) {
3661 return 0;
3662 }
3663 }
3664 }
3665 }
3666 }
3667 }
3668
3669 if (has_constants) {
3670 for (i = 0; i < parent_vars_count; i++) {
3671 int8_t reg = STACK_REG(parent_stack, i);
3672
3673 if (reg != ZREG_NONE) {
3674 if (reg < ZREG_NUM) {
3675 /* pass */
3676 } else if (reg == ZREG_THIS) {
3677 if (polymorphic_side_trace) {
3678 ssa->var_info[i].delayed_fetch_this = 1;
3679 if (stack) {
3680 SET_STACK_REG(stack, i, ZREG_THIS);
3681 }
3682 } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
3683 return 0;
3684 }
3685 } else {
3686 if (reg == ZREG_ZVAL_COPY_GPR0
3687 &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) {
3688 return 0;
3689 }
3690 if (!zend_jit_store_const(Dst, i, reg)) {
3691 return 0;
3692 }
3693 }
3694 }
3695 }
3696 }
3697
3698 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3699 if (!zend_jit_save_call_chain(Dst, -1)) {
3700 return 0;
3701 }
3702 }
3703
3704 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3705 const zend_op *op = opline - 1;
3706
3707 if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) {
3708 return 0;
3709 }
3710 }
3711
3712 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3713 const zend_op *op = opline - 1;
3714
3715 if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) {
3716 return 0;
3717 }
3718 }
3719
3720 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3721 if (!zend_jit_check_exception(Dst)) {
3722 return 0;
3723 }
3724 }
3725
3726 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3727 if (!zend_jit_free_trampoline(Dst)) {
3728 return 0;
3729 }
3730 }
3731
3732 return 1;
3733 }
3734
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3735 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3736 {
3737 info->has_range = 1;
3738 info->range.min = min;
3739 info->range.max = max;
3740 info->range.underflow = 0;
3741 info->range.overflow = 0;
3742 }
3743
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3744 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3745 {
3746 zend_long op1_min, op1_max, op2_min, op2_max;
3747
3748 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3749 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3750 return;
3751 }
3752
3753 op1_min = OP1_MIN_RANGE();
3754 op1_max = OP1_MAX_RANGE();
3755 op2_min = OP2_MIN_RANGE();
3756 op2_max = OP2_MAX_RANGE();
3757
3758 switch (opline->opcode) {
3759 case ZEND_IS_EQUAL:
3760 case ZEND_CASE:
3761 case ZEND_IS_IDENTICAL:
3762 case ZEND_CASE_STRICT:
3763 case ZEND_IS_NOT_IDENTICAL:
3764 if (!exit_if_true) {
3765 /* op1 == op2 */
3766 if (ssa_op->op1_use >= 0) {
3767 zend_jit_trace_set_var_range(
3768 &ssa->var_info[ssa_op->op1_use],
3769 MAX(op1_min, op2_min),
3770 MIN(op1_max, op2_max));
3771 }
3772 if (ssa_op->op2_use >= 0) {
3773 zend_jit_trace_set_var_range(
3774 &ssa->var_info[ssa_op->op2_use],
3775 MAX(op2_min, op1_min),
3776 MIN(op2_max, op1_max));
3777 }
3778 }
3779 break;
3780 case ZEND_IS_NOT_EQUAL:
3781 if (exit_if_true) {
3782 /* op1 == op2 */
3783 if (ssa_op->op1_use >= 0) {
3784 zend_jit_trace_set_var_range(
3785 &ssa->var_info[ssa_op->op1_use],
3786 MAX(op1_min, op2_min),
3787 MIN(op1_max, op2_max));
3788 }
3789 if (ssa_op->op2_use >= 0) {
3790 zend_jit_trace_set_var_range(
3791 &ssa->var_info[ssa_op->op2_use],
3792 MAX(op2_min, op1_min),
3793 MIN(op2_max, op1_max));
3794 }
3795 }
3796 break;
3797 case ZEND_IS_SMALLER_OR_EQUAL:
3798 if (!exit_if_true) {
3799 /* op1 <= op2 */
3800 if (ssa_op->op1_use >= 0) {
3801 zend_jit_trace_set_var_range(
3802 &ssa->var_info[ssa_op->op1_use],
3803 op1_min,
3804 MIN(op1_max, op2_max));
3805 }
3806 if (ssa_op->op2_use >= 0) {
3807 zend_jit_trace_set_var_range(
3808 &ssa->var_info[ssa_op->op2_use],
3809 MAX(op2_min, op1_min),
3810 op2_max);
3811 }
3812 } else {
3813 /* op1 > op2 */
3814 if (ssa_op->op1_use >= 0) {
3815 zend_jit_trace_set_var_range(
3816 &ssa->var_info[ssa_op->op1_use],
3817 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3818 op1_max);
3819 }
3820 if (ssa_op->op2_use >= 0) {
3821 zend_jit_trace_set_var_range(
3822 &ssa->var_info[ssa_op->op2_use],
3823 op2_min,
3824 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3825 }
3826 }
3827 break;
3828 case ZEND_IS_SMALLER:
3829 if (!exit_if_true) {
3830 /* op1 < op2 */
3831 if (ssa_op->op1_use >= 0) {
3832 zend_jit_trace_set_var_range(
3833 &ssa->var_info[ssa_op->op1_use],
3834 op1_min,
3835 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3836 }
3837 if (ssa_op->op2_use >= 0) {
3838 zend_jit_trace_set_var_range(
3839 &ssa->var_info[ssa_op->op2_use],
3840 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3841 op2_max);
3842 }
3843 } else {
3844 /* op1 >= op2 */
3845 if (ssa_op->op1_use >= 0) {
3846 zend_jit_trace_set_var_range(
3847 &ssa->var_info[ssa_op->op1_use],
3848 MAX(op1_min, op2_min),
3849 op1_max);
3850 }
3851 if (ssa_op->op2_use >= 0) {
3852 zend_jit_trace_set_var_range(
3853 &ssa->var_info[ssa_op->op2_use],
3854 op2_min,
3855 MIN(op2_max, op1_max));
3856 }
3857 }
3858 break;
3859 }
3860 }
3861
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3862 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3863 {
3864 uint8_t prev_opcode;
3865
3866 if (opline->op1_type == IS_CONST
3867 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3868 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3869 if (ssa_op->op2_use >= 0) {
3870 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3871 ssa_op--;
3872 opline = ssa_opcodes[ssa_op - ssa->ops];
3873 prev_opcode = opline->opcode;
3874 if (prev_opcode == ZEND_PRE_INC
3875 || prev_opcode == ZEND_PRE_DEC
3876 || prev_opcode == ZEND_POST_INC
3877 || prev_opcode == ZEND_POST_DEC) {
3878 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3879 }
3880 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3881 ssa_op--;
3882 opline = ssa_opcodes[ssa_op - ssa->ops];
3883 prev_opcode = opline->opcode;
3884 if (prev_opcode == ZEND_ADD
3885 || prev_opcode == ZEND_SUB) {
3886 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3887 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3888 }
3889 }
3890 }
3891 } else if (opline->op2_type == IS_CONST
3892 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3893 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3894 if (ssa_op->op1_use >= 0) {
3895 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3896 ssa_op--;
3897 opline = ssa_opcodes[ssa_op - ssa->ops];
3898 prev_opcode = opline->opcode;
3899 if (prev_opcode == ZEND_PRE_INC
3900 || prev_opcode == ZEND_PRE_DEC
3901 || prev_opcode == ZEND_POST_INC
3902 || prev_opcode == ZEND_POST_DEC) {
3903 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3904 }
3905 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3906 ssa_op--;
3907 opline = ssa_opcodes[ssa_op - ssa->ops];
3908 prev_opcode = opline->opcode;
3909 if (prev_opcode == ZEND_ADD
3910 || prev_opcode == ZEND_SUB) {
3911 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3912 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3913 }
3914 }
3915 }
3916 } else {
3917 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3918 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3919
3920 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3921 && prev_ssa_op != ssa->ops
3922 && prev_ssa_op->op1_use >= 0
3923 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3924 prev_ssa_op--;
3925 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3926 }
3927
3928 if (ssa_op->op1_use == prev_ssa_op->op1_use
3929 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3930 if (prev_opcode == ZEND_IS_EQUAL
3931 || prev_opcode == ZEND_IS_NOT_EQUAL
3932 || prev_opcode == ZEND_IS_SMALLER
3933 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3934 || prev_opcode == ZEND_CASE
3935 || prev_opcode == ZEND_IS_IDENTICAL
3936 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3937 || prev_opcode == ZEND_CASE_STRICT) {
3938 if (ssa_op->op1_use < 0) {
3939 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3940 return 0;
3941 }
3942 }
3943 if (ssa_op->op2_use < 0) {
3944 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3945 return 0;
3946 }
3947 }
3948 return 1;
3949 }
3950 }
3951 }
3952 return 0;
3953 }
3954
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3955 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3956 zend_jit_trace_rec *p,
3957 zend_jit_trace_stack_frame *frame)
3958 {
3959 if (opline->result_type == IS_TMP_VAR
3960 && (p+1)->op == ZEND_JIT_TRACE_VM
3961 && (p+1)->opline == opline + 1
3962 && ((opline+1)->opcode == ZEND_SEND_VAL
3963 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3964 && frame
3965 && frame->call
3966 && frame->call->func
3967 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3968 && (opline+1)->op1_type == IS_TMP_VAR
3969 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3970 && (opline+1)->op1.var == opline->result.var) {
3971
3972 if (frame->call && frame->call->func) {
3973 uint8_t res_type = (p+1)->op1_type;
3974
3975 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3976 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3977 }
3978 }
3979 return 1;
3980 }
3981 return 0;
3982 }
3983
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)3984 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
3985 {
3986 const void *handler = NULL;
3987 dasm_State* dasm_state = NULL;
3988 zend_script *script = NULL;
3989 zend_lifetime_interval **ra = NULL;
3990 zend_string *name = NULL;
3991 void *checkpoint;
3992 const zend_op_array *op_array;
3993 zend_ssa *ssa, *op_array_ssa;
3994 const zend_op **ssa_opcodes;
3995 zend_jit_trace_rec *p;
3996 zend_jit_op_array_trace_extension *jit_extension;
3997 int num_op_arrays = 0;
3998 zend_jit_trace_info *t;
3999 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
4000 uint8_t smart_branch_opcode;
4001 const void *exit_addr;
4002 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info;
4003 bool send_result = 0;
4004 bool skip_comparison;
4005 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
4006 zend_class_entry *ce;
4007 bool ce_is_instanceof;
4008 bool on_this = 0;
4009 bool delayed_fetch_this = 0;
4010 bool avoid_refcounting = 0;
4011 bool polymorphic_side_trace =
4012 parent_trace &&
4013 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
4014 uint32_t i;
4015 zend_jit_trace_stack_frame *frame, *top, *call;
4016 zend_jit_trace_stack *stack;
4017 uint8_t res_type = IS_UNKNOWN;
4018 const zend_op *opline, *orig_opline;
4019 const zend_ssa_op *ssa_op, *orig_ssa_op;
4020 int checked_stack;
4021 int peek_checked_stack;
4022 uint32_t frame_flags = 0;
4023
4024 JIT_G(current_trace) = trace_buffer;
4025
4026 checkpoint = zend_arena_checkpoint(CG(arena));
4027
4028 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4029
4030 if (!ssa) {
4031 goto jit_cleanup;
4032 }
4033
4034 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4035
4036 /* Register allocation */
4037 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4038 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4039 ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4040 }
4041
4042 p = trace_buffer;
4043 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4044 op_array = p->op_array;
4045 frame = JIT_G(current_frame);
4046 top = zend_jit_trace_call_frame(frame, op_array);
4047 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4048 frame->used_stack = checked_stack = peek_checked_stack = 0;
4049 stack = frame->stack;
4050 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4051 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4052 }
4053
4054 opline = p[1].opline;
4055 name = zend_jit_trace_name(op_array, opline->lineno);
4056 p += ZEND_JIT_TRACE_START_REC_SIZE;
4057
4058 dasm_init(&dasm_state, DASM_MAXSECTION);
4059 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4060 dasm_setup(&dasm_state, dasm_actions);
4061
4062 jit_extension =
4063 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4064 op_array_ssa = &jit_extension->func_info.ssa;
4065
4066 dasm_growpc(&dasm_state, 2); /* =>0: loop header */
4067 /* =>1: end of code */
4068
4069 zend_jit_align_func(&dasm_state);
4070 if (!parent_trace) {
4071 zend_jit_prologue(&dasm_state);
4072 }
4073 zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM,
4074 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4075
4076 if (!parent_trace) {
4077 zend_jit_set_last_valid_opline(opline);
4078 zend_jit_track_last_valid_opline();
4079 } else {
4080 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4081 zend_jit_trace_opline_guard(&dasm_state, opline);
4082 } else {
4083 zend_jit_reset_last_valid_opline();
4084 }
4085 }
4086
4087 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4088 int last_var;
4089 int parent_vars_count = 0;
4090 zend_jit_trace_stack *parent_stack = NULL;
4091 int used_stack = ((zend_tssa*)ssa)->used_stack;
4092
4093 if (used_stack > 0) {
4094 peek_checked_stack = used_stack;
4095 if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) {
4096 goto jit_failure;
4097 }
4098 }
4099
4100 if (parent_trace) {
4101 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4102 op_array->last_var + op_array->T);
4103 if (parent_vars_count) {
4104 parent_stack =
4105 zend_jit_traces[parent_trace].stack_map +
4106 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4107 }
4108 }
4109
4110 last_var = op_array->last_var;
4111 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4112 last_var += op_array->T;
4113 }
4114
4115 for (i = 0; i < last_var; i++) {
4116 uint32_t info = ssa->var_info[i].type;
4117
4118 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4119 uint8_t type, mem_type;
4120
4121 type = concrete_type(info);
4122 if (i < parent_vars_count
4123 && STACK_TYPE(parent_stack, i) == type) {
4124 mem_type = STACK_MEM_TYPE(parent_stack, i);
4125 if (mem_type != IS_UNKNOWN) {
4126 SET_STACK_TYPE(stack, i, mem_type, 1);
4127 }
4128 SET_STACK_TYPE(stack, i, type, 0);
4129 } else {
4130 SET_STACK_TYPE(stack, i, type, 1);
4131 }
4132 } else if (ssa->vars[i].alias != NO_ALIAS) {
4133 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4134 } else if (i < parent_vars_count
4135 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4136 /* This must be already handled by trace type inference */
4137 ZEND_UNREACHABLE();
4138 // SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
4139 } else if ((info & MAY_BE_GUARD) != 0
4140 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4141 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4142 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4143 && (opline-1)->result_type == IS_VAR
4144 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4145 && (ssa->vars[i].use_chain != -1
4146 || (ssa->vars[i].phi_use_chain
4147 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4148 /* Check loop-invariant variable type */
4149 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4150 goto jit_failure;
4151 }
4152 info &= ~MAY_BE_GUARD;
4153 ssa->var_info[i].type = info;
4154 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4155 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4156 && op_array->function_name
4157 && i >= op_array->num_args) {
4158 /* This must be already handled by trace type inference */
4159 ZEND_UNREACHABLE();
4160 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4161 }
4162
4163 if ((info & MAY_BE_PACKED_GUARD) != 0
4164 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4165 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4166 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4167 && (ssa->vars[i].use_chain != -1
4168 || (ssa->vars[i].phi_use_chain
4169 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4170 if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) {
4171 goto jit_failure;
4172 }
4173 info &= ~MAY_BE_PACKED_GUARD;
4174 ssa->var_info[i].type = info;
4175 }
4176 }
4177
4178 if (parent_trace) {
4179 /* Deoptimization */
4180 if (!zend_jit_trace_deoptimization(&dasm_state,
4181 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4182 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4183 parent_stack, parent_vars_count, ssa, stack, ra,
4184 polymorphic_side_trace)) {
4185 goto jit_failure;
4186 }
4187 }
4188
4189 if (ra
4190 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4191 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4192 for (i = 0; i < last_var; i++) {
4193 if (ra[i]
4194 && (ra[i]->flags & ZREG_LOAD) != 0
4195 && ra[i]->reg != stack[i].reg) {
4196
4197 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4198 uint8_t op_type;
4199
4200 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4201 op_type = concrete_type(ssa->var_info[i].type);
4202 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
4203 goto jit_failure;
4204 }
4205 SET_STACK_TYPE(stack, i, op_type, 1);
4206 }
4207
4208 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4209 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4210 goto jit_failure;
4211 }
4212 }
4213 }
4214 }
4215 }
4216
4217 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4218 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4219 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4220
4221 zend_jit_label(&dasm_state, 0); /* start of of trace loop */
4222
4223 if (ra) {
4224 zend_ssa_phi *phi = ssa->blocks[1].phis;
4225
4226 while (phi) {
4227 zend_lifetime_interval *ival = ra[phi->ssa_var];
4228
4229 if (ival) {
4230 if (ival->flags & ZREG_LOAD) {
4231 uint32_t info = ssa->var_info[phi->ssa_var].type;
4232 ZEND_ASSERT(ival->reg != ZREG_NONE);
4233
4234 if (info & MAY_BE_GUARD) {
4235 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4236 goto jit_failure;
4237 }
4238 info &= ~MAY_BE_GUARD;
4239 ssa->var_info[phi->ssa_var].type = info;
4240 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4241 }
4242 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_LOAD);
4243 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
4244 goto jit_failure;
4245 }
4246 } else if (ival->flags & ZREG_STORE) {
4247 ZEND_ASSERT(ival->reg != ZREG_NONE);
4248
4249 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_STORE);
4250 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg,
4251 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4252 goto jit_failure;
4253 }
4254 } else {
4255 /* Register has to be written back on side exit */
4256 SET_STACK_REG(stack, phi->var, ival->reg);
4257 }
4258 }
4259 phi = phi->next;
4260 }
4261 }
4262
4263 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4264 // if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4265 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4266 //
4267 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4268 // if (!timeout_exit_addr) {
4269 // goto jit_failure;
4270 // }
4271 // }
4272 // }
4273
4274 if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
4275 int last_var = op_array->last_var;
4276
4277 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4278 last_var += op_array->T;
4279 }
4280 for (i = 0; i < last_var; i++) {
4281 if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) {
4282 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4283 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4284 goto jit_failure;
4285 }
4286 }
4287 }
4288 }
4289 }
4290
4291 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4292 for (;;p++) {
4293 if (p->op == ZEND_JIT_TRACE_VM) {
4294 uint8_t op1_type = p->op1_type;
4295 uint8_t op2_type = p->op2_type;
4296 uint8_t op3_type = p->op3_type;
4297 uint8_t orig_op1_type = op1_type;
4298 uint8_t orig_op2_type = op2_type;
4299 uint8_t val_type = IS_UNKNOWN;
4300 bool op1_indirect;
4301 zend_class_entry *op1_ce = NULL;
4302 zend_class_entry *op2_ce = NULL;
4303 bool gen_handler = false;
4304
4305 opline = p->opline;
4306 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4307 op1_type = IS_UNKNOWN;
4308 }
4309 if (op1_type != IS_UNKNOWN) {
4310 op1_type &= ~IS_TRACE_PACKED;
4311 }
4312 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4313 op2_type = IS_UNKNOWN;
4314 }
4315 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4316 op3_type = IS_UNKNOWN;
4317 }
4318
4319 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4320 op1_ce = (zend_class_entry*)(p+1)->ce;
4321 p++;
4322 }
4323 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4324 op2_ce = (zend_class_entry*)(p+1)->ce;
4325 p++;
4326 }
4327 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4328 val_type = (p+1)->op1_type;
4329 p++;
4330 }
4331
4332 frame_flags = 0;
4333
4334 switch (opline->opcode) {
4335 case ZEND_INIT_FCALL:
4336 case ZEND_INIT_FCALL_BY_NAME:
4337 case ZEND_INIT_NS_FCALL_BY_NAME:
4338 case ZEND_INIT_METHOD_CALL:
4339 case ZEND_INIT_DYNAMIC_CALL:
4340 case ZEND_INIT_STATIC_METHOD_CALL:
4341 case ZEND_INIT_USER_CALL:
4342 case ZEND_NEW:
4343 frame->call_level++;
4344 }
4345
4346 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4347 gen_handler = 0;
4348 switch (opline->opcode) {
4349 case ZEND_PRE_INC:
4350 case ZEND_PRE_DEC:
4351 case ZEND_POST_INC:
4352 case ZEND_POST_DEC:
4353 if (opline->op1_type != IS_CV) {
4354 break;
4355 }
4356 op1_info = OP1_INFO();
4357 CHECK_OP1_TRACE_TYPE();
4358 if (!(op1_info & MAY_BE_LONG)) {
4359 break;
4360 }
4361 if (opline->result_type != IS_UNUSED) {
4362 res_use_info = zend_jit_trace_type_to_info(
4363 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4364 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4365 res_info = RES_INFO();
4366 res_addr = RES_REG_ADDR();
4367 } else {
4368 res_use_info = -1;
4369 res_info = -1;
4370 res_addr = 0;
4371 }
4372 op1_def_info = OP1_DEF_INFO();
4373 if (op1_def_info & MAY_BE_GUARD
4374 && !has_concrete_type(op1_def_info)) {
4375 op1_def_info &= ~MAY_BE_GUARD;
4376 }
4377 if (!zend_jit_inc_dec(&dasm_state, opline,
4378 op1_info, OP1_REG_ADDR(),
4379 op1_def_info, OP1_DEF_REG_ADDR(),
4380 res_use_info, res_info,
4381 res_addr,
4382 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4383 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4384 goto jit_failure;
4385 }
4386 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4387 && !(op1_info & MAY_BE_STRING)) {
4388 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4389 if (opline->result_type != IS_UNUSED) {
4390 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4391 }
4392 }
4393 if (opline->result_type != IS_UNUSED
4394 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4395 && !(op1_info & MAY_BE_STRING)) {
4396 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4397 }
4398 goto done;
4399 case ZEND_BW_OR:
4400 case ZEND_BW_AND:
4401 case ZEND_BW_XOR:
4402 case ZEND_SL:
4403 case ZEND_SR:
4404 case ZEND_MOD:
4405 op1_info = OP1_INFO();
4406 CHECK_OP1_TRACE_TYPE();
4407 op2_info = OP2_INFO();
4408 CHECK_OP2_TRACE_TYPE();
4409 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4410 break;
4411 }
4412 if (!(op1_info & MAY_BE_LONG)
4413 || !(op2_info & MAY_BE_LONG)) {
4414 break;
4415 }
4416 res_addr = RES_REG_ADDR();
4417 if (Z_MODE(res_addr) != IS_REG
4418 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4419 send_result = 1;
4420 res_use_info = -1;
4421 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4422 if (!zend_jit_reuse_ip(&dasm_state)) {
4423 goto jit_failure;
4424 }
4425 } else {
4426 res_use_info = zend_jit_trace_type_to_info(
4427 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4428 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4429 }
4430 res_info = RES_INFO();
4431 if (!zend_jit_long_math(&dasm_state, opline,
4432 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4433 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4434 res_use_info, res_info, res_addr,
4435 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4436 goto jit_failure;
4437 }
4438 goto done;
4439 case ZEND_ADD:
4440 case ZEND_SUB:
4441 case ZEND_MUL:
4442 // case ZEND_DIV: // TODO: check for division by zero ???
4443 op1_info = OP1_INFO();
4444 op1_addr = OP1_REG_ADDR();
4445 op2_info = OP2_INFO();
4446 op2_addr = OP2_REG_ADDR();
4447 if (orig_op1_type != IS_UNKNOWN
4448 && (orig_op1_type & IS_TRACE_REFERENCE)
4449 && opline->op1_type == IS_CV
4450 && (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_FCARG1)
4451 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4452 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4453 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4454 goto jit_failure;
4455 }
4456 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4457 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4458 }
4459 } else {
4460 CHECK_OP1_TRACE_TYPE();
4461 }
4462 if (orig_op2_type != IS_UNKNOWN
4463 && (orig_op2_type & IS_TRACE_REFERENCE)
4464 && opline->op2_type == IS_CV
4465 && (Z_MODE(op1_addr) != IS_REG || Z_REG(op1_addr) != ZREG_FCARG1)
4466 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4467 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op2_type, &op2_info, &op2_addr,
4468 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4469 goto jit_failure;
4470 }
4471 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4472 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4473 }
4474 } else {
4475 CHECK_OP2_TRACE_TYPE();
4476 }
4477 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4478 break;
4479 }
4480 if (opline->opcode == ZEND_ADD &&
4481 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4482 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4483 /* pass */
4484 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4485 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4486 break;
4487 }
4488 res_addr = RES_REG_ADDR();
4489 if (Z_MODE(res_addr) != IS_REG
4490 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4491 send_result = 1;
4492 res_use_info = -1;
4493 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4494 if (!zend_jit_reuse_ip(&dasm_state)) {
4495 goto jit_failure;
4496 }
4497 } else {
4498 res_use_info = zend_jit_trace_type_to_info(
4499 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4500 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4501 }
4502 res_info = RES_INFO();
4503 if (opline->opcode == ZEND_ADD &&
4504 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4505 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4506 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4507 goto jit_failure;
4508 }
4509 } else {
4510 if (!zend_jit_math(&dasm_state, opline,
4511 op1_info, op1_addr,
4512 op2_info, op2_addr,
4513 res_use_info, res_info, res_addr,
4514 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4515 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4516 goto jit_failure;
4517 }
4518 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4519 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4520 && has_concrete_type(op1_info)
4521 && has_concrete_type(op2_info)) {
4522 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4523 }
4524 }
4525 goto done;
4526 case ZEND_CONCAT:
4527 case ZEND_FAST_CONCAT:
4528 op1_info = OP1_INFO();
4529 CHECK_OP1_TRACE_TYPE();
4530 op2_info = OP2_INFO();
4531 CHECK_OP2_TRACE_TYPE();
4532 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4533 break;
4534 }
4535 if (!(op1_info & MAY_BE_STRING) ||
4536 !(op2_info & MAY_BE_STRING)) {
4537 break;
4538 }
4539 res_addr = RES_REG_ADDR();
4540 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4541 send_result = 1;
4542 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4543 if (!zend_jit_reuse_ip(&dasm_state)) {
4544 goto jit_failure;
4545 }
4546 }
4547 if (!zend_jit_concat(&dasm_state, opline,
4548 op1_info, op2_info, res_addr,
4549 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4550 goto jit_failure;
4551 }
4552 goto done;
4553 case ZEND_ASSIGN_OP:
4554 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4555 break;
4556 }
4557 op1_info = OP1_INFO();
4558 CHECK_OP1_TRACE_TYPE();
4559 op2_info = OP2_INFO();
4560 CHECK_OP2_TRACE_TYPE();
4561 if (!zend_jit_supported_binary_op(
4562 opline->extended_value, op1_info, op2_info)) {
4563 break;
4564 }
4565 op1_def_info = OP1_DEF_INFO();
4566 if (op1_def_info & MAY_BE_GUARD
4567 && !has_concrete_type(op1_def_info)) {
4568 op1_def_info &= ~MAY_BE_GUARD;
4569 }
4570 if (!zend_jit_assign_op(&dasm_state, opline,
4571 op1_info, op1_def_info, OP1_RANGE(),
4572 op2_info, OP2_RANGE(),
4573 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4574 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4575 goto jit_failure;
4576 }
4577 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4578 && has_concrete_type(op1_info)
4579 && has_concrete_type(op2_info)) {
4580 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4581 if (opline->result_type != IS_UNUSED) {
4582 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4583 }
4584 }
4585 goto done;
4586 case ZEND_ASSIGN_DIM_OP:
4587 if (opline->result_type != IS_UNUSED) {
4588 break;
4589 }
4590 if (!zend_jit_supported_binary_op(
4591 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4592 break;
4593 }
4594 if (opline->op1_type == IS_CV
4595 && (opline+1)->op1_type == IS_CV
4596 && (opline+1)->op1.var == opline->op1.var) {
4597 /* skip $a[x] += $a; */
4598 break;
4599 }
4600 op1_info = OP1_INFO();
4601 op1_addr = OP1_REG_ADDR();
4602 if (opline->op1_type == IS_VAR) {
4603 if (orig_op1_type != IS_UNKNOWN
4604 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4605 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4606 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4607 goto jit_failure;
4608 }
4609 } else {
4610 break;
4611 }
4612 }
4613 if (orig_op1_type != IS_UNKNOWN
4614 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4615 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4616 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4617 goto jit_failure;
4618 }
4619 if (opline->op1_type == IS_CV
4620 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4621 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4622 }
4623 } else {
4624 CHECK_OP1_TRACE_TYPE();
4625 }
4626 op2_info = OP2_INFO();
4627 CHECK_OP2_TRACE_TYPE();
4628 op1_data_info = OP1_DATA_INFO();
4629 CHECK_OP1_DATA_TRACE_TYPE();
4630 op1_def_info = OP1_DEF_INFO();
4631 if (!zend_jit_assign_dim_op(&dasm_state, opline,
4632 op1_info, op1_def_info, op1_addr, op2_info,
4633 op1_data_info, OP1_DATA_RANGE(), val_type,
4634 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4635 goto jit_failure;
4636 }
4637 goto done;
4638 case ZEND_PRE_INC_OBJ:
4639 case ZEND_PRE_DEC_OBJ:
4640 case ZEND_POST_INC_OBJ:
4641 case ZEND_POST_DEC_OBJ:
4642 if (opline->op2_type != IS_CONST
4643 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4644 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4645 break;
4646 }
4647 ce = NULL;
4648 ce_is_instanceof = 0;
4649 on_this = delayed_fetch_this = 0;
4650 op1_indirect = 0;
4651 if (opline->op1_type == IS_UNUSED) {
4652 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4653 ce = op_array->scope;
4654 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4655 op1_addr = 0;
4656 on_this = 1;
4657 } else {
4658 if (ssa_op->op1_use >= 0) {
4659 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4660 }
4661 op1_info = OP1_INFO();
4662 if (!(op1_info & MAY_BE_OBJECT)) {
4663 break;
4664 }
4665 op1_addr = OP1_REG_ADDR();
4666 if (opline->op1_type == IS_VAR) {
4667 if (orig_op1_type != IS_UNKNOWN
4668 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4669 op1_indirect = 1;
4670 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4671 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4672 goto jit_failure;
4673 }
4674 }
4675 }
4676 if (orig_op1_type != IS_UNKNOWN
4677 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4678 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4679 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4680 goto jit_failure;
4681 }
4682 if (opline->op1_type == IS_CV
4683 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4684 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4685 }
4686 } else {
4687 CHECK_OP1_TRACE_TYPE();
4688 }
4689 if (!(op1_info & MAY_BE_OBJECT)) {
4690 break;
4691 }
4692 if (ssa->var_info && ssa->ops) {
4693 if (ssa_op->op1_use >= 0) {
4694 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4695 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4696 ce = op1_ssa->ce;
4697 ce_is_instanceof = op1_ssa->is_instanceof;
4698 }
4699 }
4700 }
4701 if (delayed_fetch_this) {
4702 on_this = 1;
4703 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4704 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4705 } else if (op_array_ssa->ops
4706 && op_array_ssa->vars
4707 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4708 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4709 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4710 }
4711 }
4712 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4713 op1_info, op1_addr,
4714 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4715 val_type)) {
4716 goto jit_failure;
4717 }
4718 goto done;
4719 case ZEND_ASSIGN_OBJ_OP:
4720 if (opline->result_type != IS_UNUSED) {
4721 break;
4722 }
4723 if (opline->op2_type != IS_CONST
4724 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4725 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4726 break;
4727 }
4728 if (opline->op1_type == IS_CV
4729 && (opline+1)->op1_type == IS_CV
4730 && (opline+1)->op1.var == opline->op1.var) {
4731 /* skip $a->prop += $a; */
4732 break;
4733 }
4734 if (!zend_jit_supported_binary_op(
4735 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4736 break;
4737 }
4738 ce = NULL;
4739 ce_is_instanceof = 0;
4740 on_this = delayed_fetch_this = 0;
4741 op1_indirect = 0;
4742 if (opline->op1_type == IS_UNUSED) {
4743 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4744 ce = op_array->scope;
4745 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4746 op1_addr = 0;
4747 on_this = 1;
4748 } else {
4749 if (ssa_op->op1_use >= 0) {
4750 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4751 }
4752 op1_info = OP1_INFO();
4753 if (!(op1_info & MAY_BE_OBJECT)) {
4754 break;
4755 }
4756 op1_addr = OP1_REG_ADDR();
4757 if (opline->op1_type == IS_VAR) {
4758 if (orig_op1_type != IS_UNKNOWN
4759 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4760 op1_indirect = 1;
4761 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4762 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4763 goto jit_failure;
4764 }
4765 }
4766 }
4767 if (orig_op1_type != IS_UNKNOWN
4768 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4769 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4770 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4771 goto jit_failure;
4772 }
4773 if (opline->op1_type == IS_CV
4774 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4775 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4776 }
4777 } else {
4778 CHECK_OP1_TRACE_TYPE();
4779 }
4780 if (!(op1_info & MAY_BE_OBJECT)) {
4781 break;
4782 }
4783 if (ssa->var_info && ssa->ops) {
4784 if (ssa_op->op1_use >= 0) {
4785 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4786 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4787 ce = op1_ssa->ce;
4788 ce_is_instanceof = op1_ssa->is_instanceof;
4789 }
4790 }
4791 }
4792 if (delayed_fetch_this) {
4793 on_this = 1;
4794 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4795 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4796 } else if (op_array_ssa->ops
4797 && op_array_ssa->vars
4798 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4799 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4800 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4801 }
4802 }
4803 op1_data_info = OP1_DATA_INFO();
4804 CHECK_OP1_DATA_TRACE_TYPE();
4805 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
4806 op1_info, op1_addr, op1_data_info, OP1_DATA_RANGE(),
4807 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4808 val_type)) {
4809 goto jit_failure;
4810 }
4811 goto done;
4812 case ZEND_ASSIGN_OBJ:
4813 if (opline->op2_type != IS_CONST
4814 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4815 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4816 break;
4817 }
4818 ce = NULL;
4819 ce_is_instanceof = 0;
4820 on_this = delayed_fetch_this = 0;
4821 op1_indirect = 0;
4822 if (opline->op1_type == IS_UNUSED) {
4823 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4824 ce = op_array->scope;
4825 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4826 op1_addr = 0;
4827 on_this = 1;
4828 } else {
4829 if (ssa_op->op1_use >= 0) {
4830 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4831 }
4832 op1_info = OP1_INFO();
4833 if (!(op1_info & MAY_BE_OBJECT)) {
4834 break;
4835 }
4836 op1_addr = OP1_REG_ADDR();
4837 if (opline->op1_type == IS_VAR) {
4838 if (orig_op1_type != IS_UNKNOWN
4839 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4840 op1_indirect = 1;
4841 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4842 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4843 goto jit_failure;
4844 }
4845 }
4846 }
4847 if (orig_op1_type != IS_UNKNOWN
4848 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4849 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4850 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4851 goto jit_failure;
4852 }
4853 if (opline->op1_type == IS_CV
4854 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4855 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4856 }
4857 } else {
4858 CHECK_OP1_TRACE_TYPE();
4859 }
4860 if (!(op1_info & MAY_BE_OBJECT)) {
4861 break;
4862 }
4863 if (ssa->var_info && ssa->ops) {
4864 if (ssa_op->op1_use >= 0) {
4865 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4866 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4867 ce = op1_ssa->ce;
4868 ce_is_instanceof = op1_ssa->is_instanceof;
4869 }
4870 }
4871 }
4872 if (delayed_fetch_this) {
4873 on_this = 1;
4874 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4875 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4876 } else if (op_array_ssa->ops
4877 && op_array_ssa->vars
4878 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4879 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4880 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4881 }
4882 }
4883 op1_data_info = OP1_DATA_INFO();
4884 CHECK_OP1_DATA_TRACE_TYPE();
4885 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4886 op1_info, op1_addr, op1_data_info,
4887 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4888 val_type,
4889 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4890 goto jit_failure;
4891 }
4892 if ((opline+1)->op1_type == IS_CV
4893 && (ssa_op+1)->op1_def >= 0
4894 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4895 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4896 }
4897 goto done;
4898 case ZEND_ASSIGN_DIM:
4899 op1_info = OP1_INFO();
4900 op1_addr = OP1_REG_ADDR();
4901 if (opline->op1_type == IS_CV
4902 && (opline+1)->op1_type == IS_CV
4903 && (opline+1)->op1.var == opline->op1.var) {
4904 /* skip $a[x] = $a; */
4905 break;
4906 }
4907 if (opline->op1_type == IS_VAR) {
4908 if (orig_op1_type != IS_UNKNOWN
4909 && (orig_op1_type & IS_TRACE_INDIRECT)
4910 && opline->result_type == IS_UNUSED) {
4911 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4912 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4913 goto jit_failure;
4914 }
4915 } else {
4916 break;
4917 }
4918 }
4919 if (orig_op1_type != IS_UNKNOWN
4920 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4921 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4922 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4923 goto jit_failure;
4924 }
4925 if (opline->op1_type == IS_CV
4926 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4927 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4928 }
4929 } else {
4930 CHECK_OP1_TRACE_TYPE();
4931 }
4932 op2_info = OP2_INFO();
4933 CHECK_OP2_TRACE_TYPE();
4934 op1_data_info = OP1_DATA_INFO();
4935 CHECK_OP1_DATA_TRACE_TYPE();
4936 if (!zend_jit_assign_dim(&dasm_state, opline,
4937 op1_info, op1_addr, op2_info, op1_data_info, val_type,
4938 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4939 goto jit_failure;
4940 }
4941 if ((opline+1)->op1_type == IS_CV
4942 && (ssa_op+1)->op1_def >= 0
4943 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4944 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4945 }
4946 goto done;
4947 case ZEND_ASSIGN:
4948 if (opline->op1_type != IS_CV) {
4949 break;
4950 }
4951 op2_addr = OP2_REG_ADDR();
4952 op2_info = OP2_INFO();
4953
4954 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
4955 op2_def_addr = op2_addr;
4956 } else {
4957 op2_def_addr = OP2_DEF_REG_ADDR();
4958 }
4959 CHECK_OP2_TRACE_TYPE();
4960 op1_info = OP1_INFO();
4961 op1_def_info = OP1_DEF_INFO();
4962 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
4963 if (op1_type < IS_STRING
4964 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
4965 if (!zend_jit_scalar_type_guard(&dasm_state, opline, opline->op1.var)) {
4966 goto jit_failure;
4967 }
4968 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
4969 } else {
4970 CHECK_OP1_TRACE_TYPE();
4971 }
4972 }
4973 op1_addr = OP1_REG_ADDR();
4974 op1_def_addr = OP1_DEF_REG_ADDR();
4975 if (Z_MODE(op1_def_addr) != IS_REG &&
4976 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
4977 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
4978 /* type may be not set */
4979 op1_info |= MAY_BE_NULL;
4980 }
4981 if (orig_op1_type != IS_UNKNOWN) {
4982 if (orig_op1_type & IS_TRACE_REFERENCE) {
4983 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4984 !ssa->var_info[ssa_op->op1_use].guarded_reference, 0)) {
4985 goto jit_failure;
4986 }
4987 if (opline->op1_type == IS_CV
4988 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4989 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4990 }
4991 if (opline->result_type == IS_UNUSED) {
4992 res_addr = 0;
4993 } else {
4994 res_addr = RES_REG_ADDR();
4995 if (Z_MODE(res_addr) != IS_REG
4996 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4997 send_result = 1;
4998 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4999 if (!zend_jit_reuse_ip(&dasm_state)) {
5000 goto jit_failure;
5001 }
5002 }
5003 }
5004 if (!zend_jit_assign_to_typed_ref(&dasm_state, opline, opline->op2_type, op2_addr, res_addr, 1)) {
5005 goto jit_failure;
5006 }
5007 op1_def_addr = op1_addr;
5008 op1_def_info &= ~MAY_BE_REF;
5009 } else if (op1_info & MAY_BE_REF) {
5010 if (!zend_jit_noref_guard(&dasm_state, opline, op1_addr)) {
5011 goto jit_failure;
5012 }
5013 op1_info &= ~MAY_BE_REF;
5014 op1_def_info &= ~MAY_BE_REF;
5015 }
5016 }
5017 if (opline->result_type == IS_UNUSED) {
5018 res_addr = 0;
5019 res_info = -1;
5020 } else {
5021 res_addr = RES_REG_ADDR();
5022 res_info = RES_INFO();
5023 if (Z_MODE(res_addr) != IS_REG
5024 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5025 send_result = 1;
5026 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5027 if (!zend_jit_reuse_ip(&dasm_state)) {
5028 goto jit_failure;
5029 }
5030 }
5031 }
5032 if (!zend_jit_assign(&dasm_state, opline,
5033 op1_info, op1_addr,
5034 op1_def_info, op1_def_addr,
5035 op2_info, op2_addr, op2_def_addr,
5036 res_info, res_addr,
5037 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5038 goto jit_failure;
5039 }
5040 if (ssa_op->op2_def >= 0
5041 && Z_MODE(op2_addr) == IS_REG
5042 && ssa->vars[ssa_op->op2_def].no_val) {
5043 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5044 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5045
5046 if (STACK_MEM_TYPE(stack, var_num) != type
5047 && ssa->vars[ssa_op->op2_def].use_chain < 0
5048 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5049 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5050 return 0;
5051 }
5052 SET_STACK_TYPE(stack, var_num, type, 1);
5053 }
5054 }
5055 if (opline->op2_type == IS_CV
5056 && ssa_op->op2_def >= 0
5057 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5058 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5059 }
5060 goto done;
5061 case ZEND_CAST:
5062 if (opline->extended_value != op1_type) {
5063 break;
5064 }
5065 ZEND_FALLTHROUGH;
5066 case ZEND_QM_ASSIGN:
5067 op1_addr = OP1_REG_ADDR();
5068 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5069 op1_def_addr = op1_addr;
5070 } else {
5071 op1_def_addr = OP1_DEF_REG_ADDR();
5072 }
5073 op1_info = OP1_INFO();
5074 CHECK_OP1_TRACE_TYPE();
5075 res_info = RES_INFO();
5076 res_use_info = zend_jit_trace_type_to_info(
5077 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
5078 res_addr = RES_REG_ADDR();
5079 if (Z_MODE(res_addr) != IS_REG &&
5080 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5081 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5082 /* type may be not set */
5083 res_use_info |= MAY_BE_NULL;
5084 }
5085 if (!zend_jit_qm_assign(&dasm_state, opline,
5086 op1_info, op1_addr, op1_def_addr,
5087 res_use_info, res_info, res_addr)) {
5088 goto jit_failure;
5089 }
5090 if (ssa_op->op1_def >= 0
5091 && Z_MODE(op1_addr) == IS_REG
5092 && ssa->vars[ssa_op->op1_def].no_val) {
5093 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5094 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5095
5096 if (STACK_MEM_TYPE(stack, var_num) != type
5097 && ssa->vars[ssa_op->op1_def].use_chain < 0
5098 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5099 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5100 return 0;
5101 }
5102 SET_STACK_TYPE(stack, var_num, type, 1);
5103 }
5104 }
5105 if (opline->op1_type == IS_CV
5106 && ssa_op->op1_def >= 0
5107 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5108 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5109 }
5110 goto done;
5111 case ZEND_INIT_FCALL:
5112 case ZEND_INIT_FCALL_BY_NAME:
5113 case ZEND_INIT_NS_FCALL_BY_NAME:
5114 frame_flags = TRACE_FRAME_MASK_NESTED;
5115 if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5116 goto jit_failure;
5117 }
5118 goto done;
5119 case ZEND_SEND_VAL:
5120 case ZEND_SEND_VAL_EX:
5121 if (opline->op2_type == IS_CONST) {
5122 /* Named parameters not supported in JIT */
5123 break;
5124 }
5125 if (opline->opcode == ZEND_SEND_VAL_EX
5126 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5127 break;
5128 }
5129 op1_info = OP1_INFO();
5130 CHECK_OP1_TRACE_TYPE();
5131 if (!zend_jit_send_val(&dasm_state, opline,
5132 op1_info, OP1_REG_ADDR())) {
5133 goto jit_failure;
5134 }
5135 if (frame->call && frame->call->func) {
5136 if (opline->op1_type == IS_CONST) {
5137 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5138 } else if (op1_type != IS_UNKNOWN) {
5139 if (op1_type == IS_UNDEF) {
5140 op1_type = IS_NULL;
5141 }
5142 zend_jit_trace_send_type(opline, frame->call, op1_type);
5143 }
5144 }
5145 goto done;
5146 case ZEND_SEND_REF:
5147 if (opline->op2_type == IS_CONST) {
5148 /* Named parameters not supported in JIT */
5149 break;
5150 }
5151 op1_info = OP1_INFO();
5152 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
5153 op1_info, 0)) {
5154 goto jit_failure;
5155 }
5156 if (opline->op1_type == IS_CV
5157 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5158 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5159 }
5160 goto done;
5161 case ZEND_SEND_VAR:
5162 case ZEND_SEND_VAR_EX:
5163 case ZEND_SEND_VAR_NO_REF:
5164 case ZEND_SEND_VAR_NO_REF_EX:
5165 case ZEND_SEND_FUNC_ARG:
5166 if (opline->op2_type == IS_CONST) {
5167 /* Named parameters not supported in JIT */
5168 break;
5169 }
5170 if ((opline->opcode == ZEND_SEND_VAR_EX
5171 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5172 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5173 break;
5174 }
5175 op1_addr = OP1_REG_ADDR();
5176 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5177 op1_def_addr = op1_addr;
5178 } else {
5179 op1_def_addr = OP1_DEF_REG_ADDR();
5180 }
5181 op1_info = OP1_INFO();
5182 CHECK_OP1_TRACE_TYPE();
5183 if (!zend_jit_send_var(&dasm_state, opline, op_array,
5184 op1_info, op1_addr, op1_def_addr)) {
5185 goto jit_failure;
5186 }
5187 if (ssa_op->op1_def >= 0
5188 && Z_MODE(op1_addr) == IS_REG
5189 && ssa->vars[ssa_op->op1_def].no_val) {
5190 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5191 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5192
5193 if (STACK_MEM_TYPE(stack, var_num) != type
5194 && ssa->vars[ssa_op->op1_def].use_chain < 0
5195 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5196 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5197 return 0;
5198 }
5199 SET_STACK_TYPE(stack, var_num, type, 1);
5200 }
5201 }
5202 if (opline->op1_type == IS_CV
5203 && ssa_op->op1_def >= 0
5204 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5205 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5206 }
5207 if (frame->call && frame->call->func) {
5208 if ((opline->opcode == ZEND_SEND_VAR_EX
5209 || opline->opcode == ZEND_SEND_FUNC_ARG)
5210 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5211 goto done;
5212 }
5213 if (op1_type != IS_UNKNOWN) {
5214 if (op1_type == IS_UNDEF) {
5215 op1_type = IS_NULL;
5216 }
5217 zend_jit_trace_send_type(opline, frame->call, op1_type);
5218 }
5219 }
5220 goto done;
5221 case ZEND_CHECK_FUNC_ARG:
5222 if (!JIT_G(current_frame)
5223 || !JIT_G(current_frame)->call
5224 || !JIT_G(current_frame)->call->func) {
5225 break;
5226 }
5227 if (opline->op2_type == IS_CONST
5228 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5229 /* Named parameters not supported in JIT */
5230 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5231 break;
5232 }
5233 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
5234 goto jit_failure;
5235 }
5236 goto done;
5237 case ZEND_CHECK_UNDEF_ARGS:
5238 if (JIT_G(current_frame)
5239 && JIT_G(current_frame)->call) {
5240 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5241 }
5242 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
5243 goto jit_failure;
5244 }
5245 goto done;
5246 case ZEND_DO_UCALL:
5247 case ZEND_DO_ICALL:
5248 case ZEND_DO_FCALL_BY_NAME:
5249 case ZEND_DO_FCALL:
5250 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5251 goto jit_failure;
5252 }
5253 goto done;
5254 case ZEND_IS_EQUAL:
5255 case ZEND_IS_NOT_EQUAL:
5256 case ZEND_IS_SMALLER:
5257 case ZEND_IS_SMALLER_OR_EQUAL:
5258 case ZEND_CASE:
5259 op1_info = OP1_INFO();
5260 op2_info = OP2_INFO();
5261 skip_comparison =
5262 ssa_op != ssa->ops &&
5263 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5264 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5265 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5266 CHECK_OP1_TRACE_TYPE();
5267 CHECK_OP2_TRACE_TYPE();
5268 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5269 bool exit_if_true = 0;
5270 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5271 uint32_t exit_point;
5272
5273 if (ra) {
5274 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5275 }
5276 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5277 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5278 if (!exit_addr) {
5279 goto jit_failure;
5280 }
5281 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5282 if (!zend_jit_cmp(&dasm_state, opline,
5283 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5284 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5285 RES_REG_ADDR(),
5286 zend_may_throw(opline, ssa_op, op_array, ssa),
5287 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5288 goto jit_failure;
5289 }
5290 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5291 } else {
5292 smart_branch_opcode = 0;
5293 exit_addr = NULL;
5294 if (!zend_jit_cmp(&dasm_state, opline,
5295 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5296 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5297 RES_REG_ADDR(),
5298 zend_may_throw(opline, ssa_op, op_array, ssa),
5299 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5300 goto jit_failure;
5301 }
5302 }
5303 goto done;
5304 case ZEND_IS_IDENTICAL:
5305 case ZEND_IS_NOT_IDENTICAL:
5306 case ZEND_CASE_STRICT:
5307 op1_info = OP1_INFO();
5308 op2_info = OP2_INFO();
5309 skip_comparison =
5310 ssa_op != ssa->ops &&
5311 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5312 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5313 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5314 CHECK_OP1_TRACE_TYPE();
5315 CHECK_OP2_TRACE_TYPE();
5316 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5317 bool exit_if_true = 0;
5318 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5319 uint32_t exit_point;
5320
5321 if (ra) {
5322 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5323 }
5324 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5325 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5326 if (!exit_addr) {
5327 goto jit_failure;
5328 }
5329 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5330 exit_if_true = !exit_if_true;
5331 }
5332 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5333 if (!zend_jit_identical(&dasm_state, opline,
5334 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5335 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5336 RES_REG_ADDR(),
5337 zend_may_throw(opline, ssa_op, op_array, ssa),
5338 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5339 goto jit_failure;
5340 }
5341 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5342 } else {
5343 smart_branch_opcode = 0;
5344 exit_addr = NULL;
5345 if (!zend_jit_identical(&dasm_state, opline,
5346 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5347 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5348 RES_REG_ADDR(),
5349 zend_may_throw(opline, ssa_op, op_array, ssa),
5350 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5351 goto jit_failure;
5352 }
5353 }
5354 goto done;
5355 case ZEND_DEFINED:
5356 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5357 bool exit_if_true = 0;
5358 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5359 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5360
5361 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5362 if (!exit_addr) {
5363 goto jit_failure;
5364 }
5365 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5366 } else {
5367 smart_branch_opcode = 0;
5368 exit_addr = NULL;
5369 }
5370 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5371 goto jit_failure;
5372 }
5373 goto done;
5374 case ZEND_TYPE_CHECK:
5375 if (opline->extended_value == MAY_BE_RESOURCE) {
5376 // TODO: support for is_resource() ???
5377 break;
5378 }
5379 op1_info = OP1_INFO();
5380 CHECK_OP1_TRACE_TYPE();
5381 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5382 bool exit_if_true = 0;
5383 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5384 uint32_t exit_point;
5385
5386 if (ra) {
5387 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5388 }
5389 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5390 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5391 if (!exit_addr) {
5392 goto jit_failure;
5393 }
5394 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5395 } else {
5396 smart_branch_opcode = 0;
5397 exit_addr = NULL;
5398 }
5399 if (!zend_jit_type_check(&dasm_state, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5400 goto jit_failure;
5401 }
5402 goto done;
5403 case ZEND_RETURN:
5404 op1_info = OP1_INFO();
5405 CHECK_OP1_TRACE_TYPE();
5406 if (opline->op1_type == IS_CONST) {
5407 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5408 } else if (op1_type != IS_UNKNOWN) {
5409 res_type = op1_type;
5410 }
5411 if (op_array->type == ZEND_EVAL_CODE
5412 // TODO: support for top-level code
5413 || !op_array->function_name
5414 // TODO: support for IS_UNDEF ???
5415 || (op1_info & MAY_BE_UNDEF)) {
5416 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5417 goto jit_failure;
5418 }
5419 } else {
5420 int j;
5421 int may_throw = 0;
5422 bool left_frame = 0;
5423
5424 if (!zend_jit_return(&dasm_state, opline, op_array,
5425 op1_info, OP1_REG_ADDR())) {
5426 goto jit_failure;
5427 }
5428 if (op_array->last_var > 100) {
5429 /* To many CVs to unroll */
5430 if (!zend_jit_free_cvs(&dasm_state)) {
5431 goto jit_failure;
5432 }
5433 left_frame = 1;
5434 }
5435 if (!left_frame) {
5436 for (j = 0 ; j < op_array->last_var; j++) {
5437 uint32_t info;
5438 uint8_t type;
5439
5440 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5441 type = STACK_TYPE(stack, j);
5442 info = zend_jit_trace_type_to_info_ex(type, info);
5443 if (opline->op1_type == IS_CV
5444 && EX_VAR_TO_NUM(opline->op1.var) == j
5445 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5446 if (JIT_G(current_frame)
5447 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5448 continue;
5449 } else {
5450 info |= MAY_BE_NULL;
5451 }
5452 }
5453 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5454 if (!left_frame) {
5455 left_frame = 1;
5456 if (!zend_jit_leave_frame(&dasm_state)) {
5457 goto jit_failure;
5458 }
5459 }
5460 if (!zend_jit_free_cv(&dasm_state, info, j)) {
5461 goto jit_failure;
5462 }
5463 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5464 if (info & MAY_BE_RC1) {
5465 may_throw = 1;
5466 }
5467 }
5468 }
5469 }
5470 }
5471 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
5472 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5473 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5474 goto jit_failure;
5475 }
5476 }
5477 goto done;
5478 case ZEND_BOOL:
5479 case ZEND_BOOL_NOT:
5480 op1_info = OP1_INFO();
5481 CHECK_OP1_TRACE_TYPE();
5482 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5483 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5484 -1, -1,
5485 zend_may_throw(opline, ssa_op, op_array, ssa),
5486 opline->opcode, NULL)) {
5487 goto jit_failure;
5488 }
5489 goto done;
5490 case ZEND_JMPZ:
5491 case ZEND_JMPNZ:
5492 case ZEND_JMPZ_EX:
5493 case ZEND_JMPNZ_EX:
5494 op1_info = OP1_INFO();
5495 CHECK_OP1_TRACE_TYPE();
5496 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5497 const zend_op *exit_opline = NULL;
5498 uint32_t exit_point;
5499
5500 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5501 /* taken branch */
5502 if (opline->opcode == ZEND_JMPNZ_EX) {
5503 smart_branch_opcode = ZEND_JMPZ_EX;
5504 } else if (opline->opcode == ZEND_JMPZ_EX) {
5505 smart_branch_opcode = ZEND_JMPNZ_EX;
5506 } else if (opline->opcode == ZEND_JMPNZ) {
5507 smart_branch_opcode = ZEND_JMPZ;
5508 } else {
5509 smart_branch_opcode = ZEND_JMPNZ;
5510 }
5511 exit_opline = opline + 1;
5512 } else if ((p+1)->opline == opline + 1) {
5513 /* not taken branch */
5514 smart_branch_opcode = opline->opcode;
5515 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5516 } else {
5517 ZEND_UNREACHABLE();
5518 }
5519 if (ra) {
5520 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5521 }
5522 if (!(op1_info & MAY_BE_GUARD)
5523 && has_concrete_type(op1_info)
5524 && concrete_type(op1_info) <= IS_TRUE) {
5525 /* unconditional branch */
5526 exit_addr = NULL;
5527 } else if (opline->result_type == IS_TMP_VAR) {
5528 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5529 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5530
5531 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5532 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5533 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5534 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5535 if (!exit_addr) {
5536 goto jit_failure;
5537 }
5538 } else {
5539 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5540 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5541 if (!exit_addr) {
5542 goto jit_failure;
5543 }
5544 }
5545 } else {
5546 ZEND_UNREACHABLE();
5547 }
5548 if (opline->result_type == IS_UNDEF) {
5549 res_addr = 0;
5550 } else {
5551 res_addr = RES_REG_ADDR();
5552 }
5553 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5554 op1_info, OP1_REG_ADDR(), res_addr,
5555 -1, -1,
5556 zend_may_throw(opline, ssa_op, op_array, ssa),
5557 smart_branch_opcode, exit_addr)) {
5558 goto jit_failure;
5559 }
5560 goto done;
5561 case ZEND_ISSET_ISEMPTY_CV:
5562 if ((opline->extended_value & ZEND_ISEMPTY)) {
5563 // TODO: support for empty() ???
5564 break;
5565 }
5566 op1_info = OP1_INFO();
5567 op1_addr = OP1_REG_ADDR();
5568 if (orig_op1_type != IS_UNKNOWN
5569 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5570 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5571 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5572 goto jit_failure;
5573 }
5574 if (opline->op1_type == IS_CV
5575 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5576 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5577 }
5578 } else {
5579 CHECK_OP1_TRACE_TYPE();
5580 }
5581 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5582 bool exit_if_true = 0;
5583 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5584 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5585
5586 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5587 if (!exit_addr) {
5588 goto jit_failure;
5589 }
5590 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5591 } else {
5592 smart_branch_opcode = 0;
5593 exit_addr = NULL;
5594 }
5595 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
5596 op1_info, op1_addr,
5597 smart_branch_opcode, -1, -1, exit_addr)) {
5598 goto jit_failure;
5599 }
5600 goto done;
5601 case ZEND_IN_ARRAY:
5602 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5603 break;
5604 }
5605 op1_info = OP1_INFO();
5606 op1_addr = OP1_REG_ADDR();
5607 CHECK_OP1_TRACE_TYPE();
5608 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5609 break;
5610 }
5611 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5612 bool exit_if_true = 0;
5613 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5614 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5615
5616 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5617 if (!exit_addr) {
5618 goto jit_failure;
5619 }
5620 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5621 } else {
5622 smart_branch_opcode = 0;
5623 exit_addr = NULL;
5624 }
5625 if (!zend_jit_in_array(&dasm_state, opline,
5626 op1_info, op1_addr,
5627 smart_branch_opcode, -1, -1, exit_addr)) {
5628 goto jit_failure;
5629 }
5630 goto done;
5631 case ZEND_FETCH_DIM_FUNC_ARG:
5632 if (!JIT_G(current_frame)
5633 || !JIT_G(current_frame)->call
5634 || !JIT_G(current_frame)->call->func
5635 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5636 break;
5637 }
5638 ZEND_FALLTHROUGH;
5639 case ZEND_FETCH_DIM_R:
5640 case ZEND_FETCH_DIM_IS:
5641 case ZEND_FETCH_LIST_R:
5642 op1_info = OP1_INFO();
5643 op1_addr = OP1_REG_ADDR();
5644 if (orig_op1_type != IS_UNKNOWN
5645 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5646 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5647 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5648 goto jit_failure;
5649 }
5650 if (opline->op1_type == IS_CV
5651 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5652 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5653 if (ssa_op->op1_def >= 0) {
5654 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5655 }
5656 }
5657 } else {
5658 CHECK_OP1_TRACE_TYPE();
5659 }
5660 op2_info = OP2_INFO();
5661 CHECK_OP2_TRACE_TYPE();
5662 res_info = RES_INFO();
5663 avoid_refcounting =
5664 ssa_op->op1_use >= 0 &&
5665 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5666 if (op1_info & MAY_BE_PACKED_GUARD) {
5667 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5668 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5669 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5670 && MAY_BE_PACKED(op1_info)
5671 && MAY_BE_HASH(op1_info)
5672 && orig_op1_type != IS_UNKNOWN) {
5673 op1_info |= MAY_BE_PACKED_GUARD;
5674 if (orig_op1_type & IS_TRACE_PACKED) {
5675 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5676 if (op1_type != IS_UNKNOWN) {
5677 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5678 }
5679 } else {
5680 op1_info &= ~MAY_BE_ARRAY_PACKED;
5681 if (op1_type != IS_UNKNOWN) {
5682 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5683 }
5684 }
5685 }
5686 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
5687 op1_info, op1_addr, avoid_refcounting,
5688 op2_info, res_info, RES_REG_ADDR(), val_type)) {
5689 goto jit_failure;
5690 }
5691 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5692 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5693 }
5694 goto done;
5695 case ZEND_FETCH_DIM_W:
5696 case ZEND_FETCH_DIM_RW:
5697 // case ZEND_FETCH_DIM_UNSET:
5698 case ZEND_FETCH_LIST_W:
5699 if (opline->op1_type != IS_CV
5700 && (orig_op1_type == IS_UNKNOWN
5701 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5702 break;
5703 }
5704 op1_info = OP1_INFO();
5705 op1_addr = OP1_REG_ADDR();
5706 if (opline->op1_type == IS_VAR) {
5707 if (orig_op1_type != IS_UNKNOWN
5708 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5709 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5710 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5711 goto jit_failure;
5712 }
5713 } else {
5714 break;
5715 }
5716 }
5717 if (orig_op1_type != IS_UNKNOWN
5718 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5719 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5720 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5721 goto jit_failure;
5722 }
5723 if (opline->op1_type == IS_CV
5724 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5725 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5726 }
5727 } else {
5728 CHECK_OP1_TRACE_TYPE();
5729 }
5730 op2_info = OP2_INFO();
5731 CHECK_OP2_TRACE_TYPE();
5732 op1_def_info = OP1_DEF_INFO();
5733 if (!zend_jit_fetch_dim(&dasm_state, opline,
5734 op1_info, op1_addr, op2_info, RES_REG_ADDR(), val_type)) {
5735 goto jit_failure;
5736 }
5737 if (ssa_op->result_def >= 0
5738 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5739 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5740 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5741 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5742 }
5743 goto done;
5744 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5745 if ((opline->extended_value & ZEND_ISEMPTY)) {
5746 // TODO: support for empty() ???
5747 break;
5748 }
5749 op1_info = OP1_INFO();
5750 op1_addr = OP1_REG_ADDR();
5751 if (orig_op1_type != IS_UNKNOWN
5752 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5753 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5754 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5755 goto jit_failure;
5756 }
5757 if (opline->op1_type == IS_CV
5758 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5759 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5760 }
5761 } else {
5762 CHECK_OP1_TRACE_TYPE();
5763 }
5764 op2_info = OP2_INFO();
5765 CHECK_OP2_TRACE_TYPE();
5766 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5767 bool exit_if_true = 0;
5768 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5769 uint32_t exit_point;
5770
5771 if (ra) {
5772 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5773 }
5774 if (ssa_op->op1_use >= 0
5775 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5776 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5777 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5778 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5779
5780 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5781 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5782 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5783 } else {
5784 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5785 }
5786 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5787 if (!exit_addr) {
5788 goto jit_failure;
5789 }
5790 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5791 } else {
5792 smart_branch_opcode = 0;
5793 exit_addr = NULL;
5794 }
5795 avoid_refcounting =
5796 ssa_op->op1_use >= 0 &&
5797 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5798 if (op1_info & MAY_BE_PACKED_GUARD) {
5799 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5800 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5801 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5802 && MAY_BE_PACKED(op1_info)
5803 && MAY_BE_HASH(op1_info)
5804 && orig_op1_type != IS_UNKNOWN) {
5805 op1_info |= MAY_BE_PACKED_GUARD;
5806 if (orig_op1_type & IS_TRACE_PACKED) {
5807 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5808 } else {
5809 op1_info &= ~MAY_BE_ARRAY_PACKED;
5810 }
5811 }
5812 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
5813 op1_info, op1_addr, avoid_refcounting,
5814 op2_info, val_type,
5815 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5816 smart_branch_opcode, -1, -1,
5817 exit_addr)) {
5818 goto jit_failure;
5819 }
5820 goto done;
5821 case ZEND_FETCH_OBJ_FUNC_ARG:
5822 if (!JIT_G(current_frame)
5823 || !JIT_G(current_frame)->call
5824 || !JIT_G(current_frame)->call->func
5825 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5826 break;
5827 }
5828 ZEND_FALLTHROUGH;
5829 case ZEND_FETCH_OBJ_R:
5830 case ZEND_FETCH_OBJ_IS:
5831 case ZEND_FETCH_OBJ_W:
5832 on_this = delayed_fetch_this = 0;
5833 avoid_refcounting = 0;
5834 if (opline->op2_type != IS_CONST
5835 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5836 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5837 break;
5838 }
5839 ce = NULL;
5840 ce_is_instanceof = 0;
5841 op1_indirect = 0;
5842 if (opline->op1_type == IS_UNUSED) {
5843 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
5844 ce = op_array->scope;
5845 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
5846 op1_addr = 0;
5847 on_this = 1;
5848 } else {
5849 op1_info = OP1_INFO();
5850 if (!(op1_info & MAY_BE_OBJECT)) {
5851 break;
5852 }
5853 op1_addr = OP1_REG_ADDR();
5854 if (opline->op1_type == IS_VAR
5855 && opline->opcode == ZEND_FETCH_OBJ_W) {
5856 if (orig_op1_type != IS_UNKNOWN
5857 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5858 op1_indirect = 1;
5859 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5860 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5861 goto jit_failure;
5862 }
5863 }
5864 }
5865 if (orig_op1_type != IS_UNKNOWN
5866 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5867 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5868 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5869 goto jit_failure;
5870 }
5871 if (opline->op1_type == IS_CV
5872 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5873 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
5874 }
5875 } else {
5876 CHECK_OP1_TRACE_TYPE();
5877 }
5878 if (!(op1_info & MAY_BE_OBJECT)) {
5879 break;
5880 }
5881 if (ssa->var_info && ssa->ops) {
5882 if (ssa_op->op1_use >= 0) {
5883 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5884 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5885 ce = op1_ssa->ce;
5886 ce_is_instanceof = op1_ssa->is_instanceof;
5887 }
5888 }
5889 }
5890 if (ssa_op->op1_use >= 0) {
5891 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
5892 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5893 }
5894 if (delayed_fetch_this) {
5895 on_this = 1;
5896 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
5897 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
5898 } else if (op_array_ssa->ops
5899 && op_array_ssa->vars
5900 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
5901 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
5902 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
5903 }
5904 }
5905 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
5906 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
5907 on_this, delayed_fetch_this, avoid_refcounting, op1_ce, val_type,
5908 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
5909 goto jit_failure;
5910 }
5911 goto done;
5912 case ZEND_BIND_GLOBAL:
5913 orig_opline = opline;
5914 orig_ssa_op = ssa_op;
5915 while (1) {
5916 if (!ssa->ops || !ssa->var_info) {
5917 op1_info = MAY_BE_ANY|MAY_BE_REF;
5918 } else {
5919 op1_info = OP1_INFO();
5920 }
5921 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5922 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5923 }
5924 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
5925 goto jit_failure;
5926 }
5927 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
5928 opline++;
5929 ssa_op++;
5930 } else {
5931 break;
5932 }
5933 }
5934 opline = orig_opline;
5935 ssa_op = orig_ssa_op;
5936 goto done;
5937 case ZEND_RECV:
5938 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
5939 goto jit_failure;
5940 }
5941 goto done;
5942 case ZEND_RECV_INIT:
5943 orig_opline = opline;
5944 orig_ssa_op = ssa_op;
5945 while (1) {
5946 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
5947 (opline + 1)->opcode != ZEND_RECV_INIT,
5948 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5949 goto jit_failure;
5950 }
5951 if ((opline+1)->opcode == ZEND_RECV_INIT) {
5952 opline++;
5953 ssa_op++;
5954 } else {
5955 break;
5956 }
5957 }
5958 opline = orig_opline;
5959 ssa_op = orig_ssa_op;
5960 goto done;
5961 case ZEND_FREE:
5962 case ZEND_FE_FREE:
5963 op1_info = OP1_INFO();
5964 if (!zend_jit_free(&dasm_state, opline, op1_info,
5965 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5966 goto jit_failure;
5967 }
5968 goto done;
5969 case ZEND_ECHO:
5970 op1_info = OP1_INFO();
5971 CHECK_OP1_TRACE_TYPE();
5972 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5973 break;
5974 }
5975 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
5976 goto jit_failure;
5977 }
5978 goto done;
5979 case ZEND_STRLEN:
5980 op1_info = OP1_INFO();
5981 op1_addr = OP1_REG_ADDR();
5982 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
5983 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5984 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5985 goto jit_failure;
5986 }
5987 if (opline->op1_type == IS_CV
5988 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5989 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5990 }
5991 } else {
5992 CHECK_OP1_TRACE_TYPE();
5993 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5994 break;
5995 }
5996 }
5997 if (!zend_jit_strlen(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR())) {
5998 goto jit_failure;
5999 }
6000 goto done;
6001 case ZEND_COUNT:
6002 op1_info = OP1_INFO();
6003 op1_addr = OP1_REG_ADDR();
6004 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
6005 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6006 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6007 goto jit_failure;
6008 }
6009 if (opline->op1_type == IS_CV
6010 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6011 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6012 }
6013 } else {
6014 CHECK_OP1_TRACE_TYPE();
6015 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6016 break;
6017 }
6018 }
6019 if (!zend_jit_count(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6020 goto jit_failure;
6021 }
6022 goto done;
6023 case ZEND_FETCH_THIS:
6024 delayed_fetch_this = 0;
6025 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6026 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6027 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6028 delayed_fetch_this = 1;
6029 }
6030 }
6031 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) {
6032 goto jit_failure;
6033 }
6034 goto done;
6035 case ZEND_SWITCH_LONG:
6036 case ZEND_SWITCH_STRING:
6037 case ZEND_MATCH:
6038 if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6039 goto jit_failure;
6040 }
6041 goto done;
6042 case ZEND_VERIFY_RETURN_TYPE:
6043 if (opline->op1_type == IS_UNUSED) {
6044 /* Always throws */
6045 break;
6046 }
6047 if (opline->op1_type == IS_CONST) {
6048 /* TODO Different instruction format, has return value */
6049 break;
6050 }
6051 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6052 /* Not worth bothering with */
6053 break;
6054 }
6055 op1_info = OP1_INFO();
6056 CHECK_OP1_TRACE_TYPE();
6057 if (op1_info & MAY_BE_REF) {
6058 /* TODO May need reference unwrapping. */
6059 break;
6060 }
6061 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, op1_info)) {
6062 goto jit_failure;
6063 }
6064 goto done;
6065 case ZEND_FE_RESET_R:
6066 op1_info = OP1_INFO();
6067 CHECK_OP1_TRACE_TYPE();
6068 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6069 break;
6070 }
6071 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
6072 goto jit_failure;
6073 }
6074 goto done;
6075 case ZEND_FE_FETCH_R:
6076 op1_info = OP1_INFO();
6077 CHECK_OP1_TRACE_TYPE();
6078 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6079 break;
6080 }
6081 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6082 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6083 uint32_t exit_point;
6084
6085 if ((p+1)->opline == exit_opline) {
6086 /* taken branch (exit from loop) */
6087 exit_opline = opline;
6088 smart_branch_opcode = ZEND_NOP;
6089 } else if ((p+1)->opline == opline + 1) {
6090 /* not taken branch (loop) */
6091 smart_branch_opcode = ZEND_JMP;
6092 } else {
6093 ZEND_UNREACHABLE();
6094 }
6095 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6096 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6097 if (!exit_addr) {
6098 goto jit_failure;
6099 }
6100 } else {
6101 ZEND_UNREACHABLE();
6102 }
6103 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
6104 -1, smart_branch_opcode, exit_addr)) {
6105 goto jit_failure;
6106 }
6107 goto done;
6108 case ZEND_FETCH_CONSTANT:
6109 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6110 goto jit_failure;
6111 }
6112 goto done;
6113 case ZEND_INIT_METHOD_CALL:
6114 if (opline->op2_type != IS_CONST
6115 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6116 break;
6117 }
6118 on_this = delayed_fetch_this = 0;
6119 ce = NULL;
6120 ce_is_instanceof = 0;
6121 if (opline->op1_type == IS_UNUSED) {
6122 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6123 ce = op_array->scope;
6124 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
6125 op1_addr = 0;
6126 on_this = 1;
6127 } else {
6128 op1_info = OP1_INFO();
6129 op1_addr = OP1_REG_ADDR();
6130 if (polymorphic_side_trace) {
6131 op1_info = MAY_BE_OBJECT;
6132 op1_addr = 0;
6133 } else if (orig_op1_type != IS_UNKNOWN
6134 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6135 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6136 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6137 goto jit_failure;
6138 }
6139 if (opline->op1_type == IS_CV
6140 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6141 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6142 }
6143 } else {
6144 CHECK_OP1_TRACE_TYPE();
6145 }
6146 if (ssa->var_info && ssa->ops) {
6147 if (ssa_op->op1_use >= 0) {
6148 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6149 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6150 ce = op1_ssa->ce;
6151 ce_is_instanceof = op1_ssa->is_instanceof;
6152 }
6153 }
6154 }
6155 if (ssa_op->op1_use >= 0) {
6156 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6157 }
6158 if (delayed_fetch_this) {
6159 on_this = 1;
6160 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6161 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6162 } else if (op_array_ssa->ops
6163 && op_array_ssa->vars
6164 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6165 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6166 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6167 }
6168 }
6169 frame_flags = TRACE_FRAME_MASK_NESTED;
6170 if (!zend_jit_init_method_call(&dasm_state, opline,
6171 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6172 op_array, ssa, ssa_op, frame->call_level,
6173 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6174 p + 1, peek_checked_stack - checked_stack, polymorphic_side_trace)) {
6175 goto jit_failure;
6176 }
6177 goto done;
6178 case ZEND_INIT_DYNAMIC_CALL:
6179 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6180 break;
6181 }
6182 op2_info = OP2_INFO();
6183 CHECK_OP2_TRACE_TYPE();
6184 frame_flags = TRACE_FRAME_MASK_NESTED;
6185 if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6186 goto jit_failure;
6187 }
6188 goto done;
6189 case ZEND_SEND_ARRAY:
6190 case ZEND_SEND_UNPACK:
6191 if (JIT_G(current_frame)
6192 && JIT_G(current_frame)->call) {
6193 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6194 }
6195 break;
6196 case ZEND_ROPE_INIT:
6197 case ZEND_ROPE_ADD:
6198 case ZEND_ROPE_END:
6199 op2_info = OP2_INFO();
6200 CHECK_OP2_TRACE_TYPE();
6201 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6202 break;
6203 }
6204 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
6205 goto jit_failure;
6206 }
6207 goto done;
6208 default:
6209 break;
6210 }
6211 }
6212
6213 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6214 gen_handler = 1;
6215 op1_info = OP1_INFO();
6216 op2_info = OP2_INFO();
6217 if (op1_info & MAY_BE_GUARD) {
6218 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6219 }
6220 if (op2_info & MAY_BE_GUARD) {
6221 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6222 }
6223 if (!zend_jit_trace_handler(&dasm_state, op_array, opline,
6224 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6225 goto jit_failure;
6226 }
6227 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
6228 if (opline->opcode == ZEND_NEW && ssa_op->result_def >= 0) {
6229 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6230 }
6231 if (zend_jit_may_be_polymorphic_call(opline) ||
6232 zend_jit_may_be_modified((p+1)->func, op_array)) {
6233 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
6234 goto jit_failure;
6235 }
6236 }
6237 }
6238 }
6239
6240 done:
6241 polymorphic_side_trace = 0;
6242 switch (opline->opcode) {
6243 case ZEND_DO_FCALL:
6244 case ZEND_DO_ICALL:
6245 case ZEND_DO_UCALL:
6246 case ZEND_DO_FCALL_BY_NAME:
6247 case ZEND_CALLABLE_CONVERT:
6248 frame->call_level--;
6249 }
6250
6251 if (ra) {
6252 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
6253 }
6254
6255 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6256 && STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var)) > ZREG_NUM) {
6257 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6258 }
6259
6260 if (opline->opcode == ZEND_ROPE_INIT) {
6261 /* clear stack slots used by rope */
6262 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6263 uint32_t count =
6264 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6265
6266 do {
6267 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6268 var++;
6269 count--;
6270 } while (count);
6271 }
6272
6273 if (ssa_op) {
6274 zend_ssa_range tmp;
6275
6276 /* Keep information about known types on abstract stack */
6277 if (ssa_op->result_def >= 0) {
6278 uint8_t type = IS_UNKNOWN;
6279
6280 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6281 || send_result) {
6282 /* we didn't set result variable */
6283 type = IS_UNKNOWN;
6284 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6285 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6286 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6287 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6288 if (opline->op1_type != IS_CONST) {
6289 /* copy */
6290 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6291 }
6292 } else if (opline->opcode == ZEND_ASSIGN) {
6293 if (opline->op2_type != IS_CONST
6294 && ssa_op->op1_use >= 0
6295 /* assignment to typed reference may cause conversion */
6296 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6297 /* copy */
6298 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6299 }
6300 } else if (opline->opcode == ZEND_POST_INC
6301 || opline->opcode == ZEND_POST_DEC) {
6302 /* copy */
6303 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6304 }
6305 if (opline->opcode == ZEND_JMP_SET
6306 || opline->opcode == ZEND_COALESCE
6307 || opline->opcode == ZEND_JMP_NULL) {
6308 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6309 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6310 } else if ((p+1)->opline != (opline + 1)) {
6311 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6312 }
6313 } else {
6314 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6315 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->result_def]));
6316 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6317 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6318 }
6319 if (type != IS_UNKNOWN) {
6320 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6321 if (opline->opcode == ZEND_FETCH_THIS
6322 && delayed_fetch_this) {
6323 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_THIS);
6324 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6325 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_TRY_ADDREF);
6326 } else if (ra && ra[ssa_op->result_def]) {
6327 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6328 ra[ssa_op->result_def]->flags & ZREG_STORE);
6329 }
6330 }
6331 }
6332
6333 if (type == IS_LONG
6334 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) {
6335 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6336 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6337 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6338 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6339 ssa->var_info[ssa_op->result_def].has_range = 1;
6340 }
6341 }
6342 if (ssa_op->op1_def >= 0
6343 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6344 || opline->result_type != IS_CV
6345 || opline->result.var != opline->op1.var)) {
6346 uint8_t type = IS_UNKNOWN;
6347
6348 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6349 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6350 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6351 } else if (opline->opcode == ZEND_ASSIGN) {
6352 if (!(OP1_INFO() & MAY_BE_REF)
6353 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6354 if (opline->op2_type != IS_CONST) {
6355 /* copy */
6356 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6357 }
6358 }
6359 } else if (opline->opcode == ZEND_SEND_VAR
6360 || opline->opcode == ZEND_CAST
6361 || opline->opcode == ZEND_QM_ASSIGN
6362 || opline->opcode == ZEND_JMP_SET
6363 || opline->opcode == ZEND_COALESCE
6364 || opline->opcode == ZEND_JMP_NULL
6365 || opline->opcode == ZEND_FE_RESET_R) {
6366 /* keep old value */
6367 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6368 }
6369 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6370 (gen_handler || type == IS_UNKNOWN || !ra ||
6371 (!ra[ssa_op->op1_def] &&
6372 !(ssa->vars[ssa_op->op1_def].no_val &&
6373 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6374 (opline->opcode == ZEND_QM_ASSIGN ||
6375 opline->opcode == ZEND_SEND_VAR ||
6376 opline->opcode == ZEND_SEND_VAR_EX ||
6377 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6378 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6379 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6380 if (type != IS_UNKNOWN) {
6381 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6382 if (ra && ra[ssa_op->op1_def]) {
6383 uint8_t flags = ra[ssa_op->op1_def]->flags & ZREG_STORE;
6384
6385 if (ssa_op->op1_use >= 0) {
6386 if (opline->opcode == ZEND_SEND_VAR
6387 || opline->opcode == ZEND_CAST
6388 || opline->opcode == ZEND_QM_ASSIGN
6389 || opline->opcode == ZEND_JMP_SET
6390 || opline->opcode == ZEND_COALESCE
6391 || opline->opcode == ZEND_JMP_NULL
6392 || opline->opcode == ZEND_FE_RESET_R) {
6393 if (!ra[ssa_op->op1_use]) {
6394 flags |= ZREG_LOAD;
6395 }
6396 }
6397 }
6398 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, flags);
6399 }
6400 }
6401 if (type == IS_LONG
6402 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6403 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6404 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6405 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6406 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6407 ssa->var_info[ssa_op->op1_def].has_range = 1;
6408 }
6409 }
6410 if (ssa_op->op2_def >= 0
6411 && (opline->opcode != ZEND_ASSIGN
6412 || opline->op1_type != IS_CV
6413 || opline->op1.var != opline->op2.var)) {
6414 uint8_t type = IS_UNKNOWN;
6415
6416 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6417 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6418 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6419 } else if (opline->opcode == ZEND_ASSIGN) {
6420 /* keep old value */
6421 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6422 }
6423 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6424 (gen_handler || type == IS_UNKNOWN || !ra ||
6425 (!ra[ssa_op->op2_def] &&
6426 !(ssa->vars[ssa_op->op2_def].no_val &&
6427 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6428 opline->opcode == ZEND_ASSIGN))));
6429 if (type != IS_UNKNOWN) {
6430 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6431 if (ra && ra[ssa_op->op2_def]) {
6432 uint8_t flags = ra[ssa_op->op2_def]->flags & ZREG_STORE;
6433
6434 if (ssa_op->op2_use >= 0) {
6435 if (opline->opcode == ZEND_ASSIGN) {
6436 if (!ra[ssa_op->op2_use]
6437 || ra[ssa_op->op2_use]->reg != ra[ssa_op->op2_def]->reg) {
6438 flags |= ZREG_LOAD;
6439 }
6440 }
6441 }
6442 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def]->reg, flags);
6443 }
6444 }
6445 if (type == IS_LONG
6446 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) {
6447 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6448 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6449 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6450 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6451 ssa->var_info[ssa_op->op2_def].has_range = 1;
6452 }
6453 }
6454
6455 switch (opline->opcode) {
6456 case ZEND_ASSIGN_DIM:
6457 case ZEND_ASSIGN_OBJ:
6458 case ZEND_ASSIGN_STATIC_PROP:
6459 case ZEND_ASSIGN_DIM_OP:
6460 case ZEND_ASSIGN_OBJ_OP:
6461 case ZEND_ASSIGN_STATIC_PROP_OP:
6462 case ZEND_ASSIGN_OBJ_REF:
6463 case ZEND_ASSIGN_STATIC_PROP_REF:
6464 /* OP_DATA */
6465 ssa_op++;
6466 opline++;
6467 if (ssa_op->op1_def >= 0) {
6468 uint8_t type = IS_UNKNOWN;
6469
6470 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6471 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6472 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6473 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6474 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6475 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6476 /* keep old value */
6477 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6478 }
6479 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6480 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->op1_def]));
6481 if (type != IS_UNKNOWN) {
6482 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6483 if (ra && ra[ssa_op->op1_def]) {
6484 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6485 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6486 }
6487 }
6488 if (type == IS_LONG
6489 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6490 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6491 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6492 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6493 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6494 ssa->var_info[ssa_op->op1_def].has_range = 1;
6495 }
6496 }
6497 ssa_op++;
6498 break;
6499 case ZEND_RECV_INIT:
6500 ssa_op++;
6501 opline++;
6502 while (opline->opcode == ZEND_RECV_INIT) {
6503 if (ssa_op->result_def >= 0) {
6504 uint8_t type = IS_UNKNOWN;
6505
6506 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6507 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6508 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6509 }
6510 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6511 (gen_handler || !ra || !ra[ssa_op->result_def]));
6512 if (ra && ra[ssa_op->result_def]) {
6513 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6514 ra[ssa_op->result_def]->flags & ZREG_STORE);
6515 }
6516 }
6517 ssa_op++;
6518 opline++;
6519 }
6520 break;
6521 case ZEND_BIND_GLOBAL:
6522 ssa_op++;
6523 opline++;
6524 while (opline->opcode == ZEND_BIND_GLOBAL) {
6525 if (ssa_op->op1_def >= 0) {
6526 uint8_t type = IS_UNKNOWN;
6527
6528 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6529 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6530 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6531 }
6532 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6533 (gen_handler || !ra || !ra[ssa_op->op1_def]));
6534 if (ra && ra[ssa_op->op1_def]) {
6535 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6536 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6537 }
6538 }
6539 ssa_op++;
6540 opline++;
6541 }
6542 break;
6543 default:
6544 ssa_op += zend_jit_trace_op_len(opline);
6545 break;
6546 }
6547
6548 if (send_result) {
6549 ssa_op++;
6550 p++;
6551 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6552 p++;
6553 }
6554 send_result = 0;
6555 }
6556 }
6557 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6558 call = frame->call;
6559 assert(call && &call->func->op_array == p->op_array);
6560
6561 if (opline->opcode == ZEND_DO_UCALL
6562 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6563 || opline->opcode == ZEND_DO_FCALL) {
6564
6565 frame->call_opline = opline;
6566
6567 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6568 if (opline > op_array->opcodes) {
6569 const zend_op *prev_opline = opline - 1;
6570
6571 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6572 prev_opline--;
6573 }
6574 JIT_G(current_frame) = call;
6575 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6576 || prev_opline->opcode == ZEND_SEND_UNPACK
6577 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6578 && p->op_array->num_args
6579 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6580 && ((p+1)->op == ZEND_JIT_TRACE_VM
6581 || (p+1)->op == ZEND_JIT_TRACE_END)
6582 && (TRACE_FRAME_NUM_ARGS(call) < 0
6583 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6584 && !zend_jit_trace_opline_guard(&dasm_state, (p+1)->opline)) {
6585 goto jit_failure;
6586 }
6587 JIT_G(current_frame) = frame;
6588 }
6589 }
6590
6591 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6592 p++;
6593 break;
6594 }
6595 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6596 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6597 TRACE_FRAME_SET_THIS_CHECKED(call);
6598 }
6599 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6600 TRACE_FRAME_SET_THIS_CHECKED(call);
6601 }
6602 op_array = (zend_op_array*)p->op_array;
6603 jit_extension =
6604 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6605 op_array_ssa = &jit_extension->func_info.ssa;
6606 frame->call = call->prev;
6607 call->prev = frame;
6608 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6609 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6610 } else {
6611 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6612 }
6613 JIT_G(current_frame) = frame = call;
6614 stack = frame->stack;
6615 if (ra) {
6616 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6617
6618 for (i = 0; i < op_array->last_var; i++,j++) {
6619 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6620 if ((ssa->var_info[j].type & MAY_BE_GUARD) != 0) {
6621 uint8_t op_type;
6622
6623 ssa->var_info[j].type &= ~MAY_BE_GUARD;
6624 op_type = concrete_type(ssa->var_info[j].type);
6625 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
6626 goto jit_failure;
6627 }
6628 SET_STACK_TYPE(stack, i, op_type, 1);
6629 }
6630 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6631 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6632 goto jit_failure;
6633 }
6634 }
6635 }
6636 }
6637 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6638 op_array = (zend_op_array*)p->op_array;
6639 jit_extension =
6640 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6641 op_array_ssa = &jit_extension->func_info.ssa;
6642 top = frame;
6643 if (frame->prev) {
6644 checked_stack = frame->old_checked_stack;
6645 peek_checked_stack = frame->old_peek_checked_stack;
6646 frame = frame->prev;
6647 stack = frame->stack;
6648 ZEND_ASSERT(&frame->func->op_array == op_array);
6649 } else {
6650 frame = zend_jit_trace_ret_frame(frame, op_array);
6651 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6652 frame->used_stack = checked_stack = peek_checked_stack = 0;
6653 stack = frame->stack;
6654 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6655 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6656
6657 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6658 /* Initialize abstract stack using SSA */
6659 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6660 && has_concrete_type(ssa->var_info[j].type)) {
6661 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6662 } else {
6663 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6664 }
6665 }
6666 if (ra) {
6667 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6668 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6669 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6670 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6671 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6672 goto jit_failure;
6673 }
6674 }
6675 }
6676 }
6677 } else {
6678 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6679 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6680 }
6681 }
6682 opline = NULL;
6683 }
6684 JIT_G(current_frame) = frame;
6685 if (res_type != IS_UNKNOWN
6686 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6687 const zend_op *opline = (p+1)->opline - 1;
6688 if (opline->result_type != IS_UNUSED) {
6689 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6690 }
6691 }
6692 res_type = IS_UNKNOWN;
6693 } else if (p->op == ZEND_JIT_TRACE_END) {
6694 break;
6695 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6696 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6697 int num_args = -1;
6698
6699 if (init_opline
6700 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6701 num_args = init_opline->extended_value;
6702 }
6703
6704 call = top;
6705 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6706 call->prev = frame->call;
6707 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6708 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6709 if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
6710 TRACE_FRAME_SET_CLOSURE_CALL(call);
6711 }
6712 }
6713 if (init_opline) {
6714 if (init_opline->opcode != ZEND_NEW
6715 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6716 || init_opline->op1_type == IS_UNDEF
6717 || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6718 && ssa_op
6719 && (ssa_op-1)->op1_use >=0
6720 && ssa->var_info[(ssa_op-1)->op1_use].delayed_fetch_this))
6721 && (init_opline->opcode != ZEND_INIT_USER_CALL
6722 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6723 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6724 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6725 ) {
6726 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6727 } else if (init_opline->opcode == ZEND_NEW
6728 || (init_opline->opcode == ZEND_INIT_METHOD_CALL
6729 && init_opline->op1_type != IS_UNDEF
6730 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6731 && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) {
6732 TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call);
6733 }
6734 }
6735 frame->call = call;
6736 top = zend_jit_trace_call_frame(top, p->op_array);
6737 if (p->func) {
6738 if (p->func->type == ZEND_USER_FUNCTION) {
6739 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6740 zend_jit_op_array_trace_extension *jit_extension =
6741 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6742
6743 i = 0;
6744 while (i < p->op_array->num_args) {
6745 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6746 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6747 i++;
6748 }
6749 while (i < p->op_array->last_var) {
6750 if (jit_extension
6751 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6752 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6753 } else {
6754 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6755 }
6756 i++;
6757 }
6758 while (i < p->op_array->last_var + p->op_array->T) {
6759 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6760 i++;
6761 }
6762 } else {
6763 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
6764 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6765 }
6766 }
6767 } else {
6768 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
6769 for (i = 0; i < p->op_array->num_args; i++) {
6770 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6771 }
6772 }
6773 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6774 int skip_guard = 0;
6775
6776 if (init_opline) {
6777 zend_call_info *call_info = jit_extension->func_info.callee_info;
6778
6779 while (call_info) {
6780 if (call_info->caller_init_opline == init_opline
6781 && !call_info->is_prototype) {
6782 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
6783 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
6784 && init_opline->op1_type != IS_CONST) {
6785 break;
6786 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
6787 break;
6788 }
6789 }
6790 skip_guard = 1;
6791 break;
6792 }
6793 call_info = call_info->next_callee;
6794 }
6795 if (!skip_guard
6796 && !zend_jit_may_be_polymorphic_call(init_opline)
6797 && !zend_jit_may_be_modified(p->func, op_array)) {
6798 skip_guard = 1;
6799 }
6800 }
6801
6802 if (!skip_guard) {
6803 if (!opline) {
6804 zend_jit_trace_rec *q = p + 1;
6805 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
6806 q++;
6807 }
6808 opline = q->opline;
6809 ZEND_ASSERT(opline != NULL);
6810 }
6811 if (!zend_jit_init_fcall_guard(&dasm_state,
6812 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
6813 goto jit_failure;
6814 }
6815 }
6816 }
6817 }
6818 call->old_checked_stack = checked_stack;
6819 call->old_peek_checked_stack = peek_checked_stack;
6820 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6821 frame->call_level++;
6822 call->used_stack = checked_stack = peek_checked_stack = 0;
6823 } else {
6824 if (p->func) {
6825 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
6826 } else {
6827 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
6828 }
6829 switch (init_opline->opcode) {
6830 case ZEND_INIT_FCALL:
6831 case ZEND_INIT_FCALL_BY_NAME:
6832 case ZEND_INIT_NS_FCALL_BY_NAME:
6833 case ZEND_INIT_METHOD_CALL:
6834 case ZEND_INIT_DYNAMIC_CALL:
6835 //case ZEND_INIT_STATIC_METHOD_CALL:
6836 //case ZEND_INIT_USER_CALL:
6837 //case ZEND_NEW:
6838 checked_stack += call->used_stack;
6839 if (checked_stack > peek_checked_stack) {
6840 peek_checked_stack = checked_stack;
6841 }
6842 break;
6843 default:
6844 checked_stack = peek_checked_stack = 0;
6845 }
6846 }
6847 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
6848 call = frame->call;
6849 if (call) {
6850 checked_stack = call->old_checked_stack;
6851 peek_checked_stack = call->old_peek_checked_stack;
6852 top = call;
6853 frame->call = call->prev;
6854 }
6855 } else {
6856 ZEND_UNREACHABLE();
6857 }
6858 }
6859
6860 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
6861
6862 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
6863
6864 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6865 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6866 }
6867
6868 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
6869 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
6870 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6871 if (ra) {
6872 zend_ssa_phi *phi = ssa->blocks[1].phis;
6873
6874 while (phi) {
6875 if (ra[phi->ssa_var]
6876 && ra[phi->sources[1]]
6877 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
6878 && (ra[phi->ssa_var]->flags & (ZREG_LOAD|ZREG_STORE)) == 0
6879 && (ra[phi->sources[1]]->flags & (ZREG_LOAD|ZREG_STORE)) == 0) {
6880 /* Store actual type to memory to avoid deoptimization mistakes */
6881 /* TODO: Alternatively, we may try to update alredy generated deoptimization info */
6882 zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var));
6883 }
6884 phi = phi->next;
6885 }
6886 }
6887 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6888 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6889 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6890 goto jit_failure;
6891 }
6892 }
6893 t->link = ZEND_JIT_TRACE_NUM;
6894 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6895 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
6896 }
6897 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
6898 const void *timeout_exit_addr = NULL;
6899
6900 t->flags |= ZEND_JIT_TRACE_LOOP;
6901
6902 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6903 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6904 || (ra
6905 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
6906 /* Deoptimize to the first instruction of the loop */
6907 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
6908
6909 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6910 if (!timeout_exit_addr) {
6911 goto jit_failure;
6912 }
6913 } else {
6914 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6915 }
6916 }
6917
6918 zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
6919 }
6920 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
6921 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
6922 if (ra
6923 && (p-1)->op != ZEND_JIT_TRACE_ENTER
6924 && (p-1)->op != ZEND_JIT_TRACE_BACK
6925 && opline->opcode != ZEND_DO_UCALL
6926 && opline->opcode != ZEND_DO_FCALL
6927 && opline->opcode != ZEND_DO_FCALL_BY_NAME
6928 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
6929 if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL,
6930 stack, op_array->last_var + op_array->T, NULL, NULL, NULL, 0)) {
6931 goto jit_failure;
6932 }
6933 for (i = 0; i < op_array->last_var; i++) {
6934 int8_t reg = STACK_REG(stack, i);
6935 uint8_t type = STACK_TYPE(stack, i);
6936
6937 if (reg == ZREG_NONE
6938 && type != IS_UNKNOWN
6939 && type != STACK_MEM_TYPE(stack, i)) {
6940 if (!zend_jit_store_var_type(&dasm_state, i, type)) {
6941 return 0;
6942 }
6943 SET_STACK_TYPE(stack, i, type, 1);
6944 }
6945 }
6946 }
6947 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
6948 const void *timeout_exit_addr = NULL;
6949
6950 t->link = zend_jit_find_trace(p->opline->handler);
6951 if (t->link == 0) {
6952 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
6953 * by zend_jit_trace_exit() in another thread after this
6954 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
6955 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
6956 * the "_counter_handler" functions, and these are not registered
6957 * tracer functions */
6958 goto jit_failure;
6959 }
6960 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6961 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6962 goto jit_failure;
6963 }
6964 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6965 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6966 }
6967 if (parent_trace
6968 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
6969 && zend_jit_traces[parent_trace].root == t->link) {
6970 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
6971 uint32_t exit_point;
6972
6973 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6974 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6975 }
6976 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
6977 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6978 if (!timeout_exit_addr) {
6979 goto jit_failure;
6980 }
6981 } else {
6982 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6983 }
6984 }
6985 zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
6986 } else {
6987 zend_jit_trace_return(&dasm_state, 0, NULL);
6988 }
6989 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
6990 zend_jit_trace_return(&dasm_state, 0, NULL);
6991 } else {
6992 // TODO: not implemented ???
6993 ZEND_ASSERT(0 && p->stop);
6994 }
6995
6996 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
6997 goto jit_failure;
6998 }
6999
7000 if (!zend_jit_trace_end(&dasm_state, t)) {
7001 goto jit_failure;
7002 }
7003
7004 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM,
7005 parent_trace ? SP_ADJ_JIT : ((zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET),
7006 parent_trace ? SP_ADJ_NONE : SP_ADJ_JIT);
7007
7008 if (handler) {
7009 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
7010 const zend_op_array *rec_op_array;
7011
7012 rec_op_array = op_array = trace_buffer->op_array;
7013 jit_extension =
7014 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7015 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7016 for (;;p++) {
7017 if (p->op == ZEND_JIT_TRACE_VM) {
7018 opline = p->opline;
7019 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7020 if (p->op_array == rec_op_array) {
7021 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7022 }
7023 op_array = p->op_array;
7024 jit_extension =
7025 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7026 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7027 op_array = p->op_array;
7028 jit_extension =
7029 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7030 } else if (p->op == ZEND_JIT_TRACE_END) {
7031 break;
7032 }
7033 }
7034 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7035 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7036 if (opline
7037 && (opline->opcode == ZEND_DO_UCALL
7038 || opline->opcode == ZEND_DO_FCALL
7039 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7040 || opline->opcode == ZEND_YIELD
7041 || opline->opcode == ZEND_YIELD_FROM
7042 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7043 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7044 }
7045 if (JIT_G(current_frame)
7046 && JIT_G(current_frame)->prev) {
7047 frame = JIT_G(current_frame)->prev;
7048 do {
7049 if (frame->call_opline) {
7050 op_array = &frame->func->op_array;
7051 jit_extension =
7052 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7053 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7054 }
7055 frame = frame->prev;
7056 } while (frame);
7057 }
7058 }
7059 }
7060
7061 jit_failure:
7062 dasm_free(&dasm_state);
7063
7064 if (name) {
7065 zend_string_release(name);
7066 }
7067
7068 jit_cleanup:
7069 /* Clean up used op_arrays */
7070 while (num_op_arrays > 0) {
7071 op_array = op_arrays[--num_op_arrays];
7072 jit_extension =
7073 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7074
7075 jit_extension->func_info.num = 0;
7076 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7077 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7078 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7079 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7080 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7081 }
7082
7083 zend_arena_release(&CG(arena), checkpoint);
7084
7085 JIT_G(current_frame) = NULL;
7086 JIT_G(current_trace) = NULL;
7087
7088 return handler;
7089 }
7090
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7091 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7092 {
7093 const void *handler = NULL;
7094 dasm_State* dasm_state = NULL;
7095 void *checkpoint;
7096 char name[32];
7097 const zend_op *opline;
7098 uint32_t stack_size;
7099 zend_jit_trace_stack *stack;
7100 bool original_handler = 0;
7101
7102 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7103 return dasm_labels[zend_lbtrace_escape];
7104 }
7105
7106 checkpoint = zend_arena_checkpoint(CG(arena));;
7107
7108 sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num);
7109
7110 dasm_init(&dasm_state, DASM_MAXSECTION);
7111 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
7112 dasm_setup(&dasm_state, dasm_actions);
7113
7114 zend_jit_align_func(&dasm_state);
7115
7116 /* Deoptimization */
7117 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7118 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
7119
7120 if (!zend_jit_trace_deoptimization(&dasm_state,
7121 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7122 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7123 stack, stack_size, NULL, NULL, NULL, 0)) {
7124 goto jit_failure;
7125 }
7126
7127 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7128 if (opline) {
7129 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7130 /* prevent endless loop */
7131 original_handler = 1;
7132 }
7133 zend_jit_set_ip_ex(&dasm_state, opline, original_handler);
7134 }
7135
7136 zend_jit_trace_return(&dasm_state, original_handler, opline);
7137
7138 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM, SP_ADJ_JIT, SP_ADJ_NONE);
7139
7140 jit_failure:
7141 dasm_free(&dasm_state);
7142 zend_arena_release(&CG(arena), checkpoint);
7143 return handler;
7144 }
7145
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7146 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7147 {
7148 zend_jit_trace_stop ret;
7149 const void *handler;
7150 uint8_t orig_trigger;
7151 zend_jit_trace_info *t = NULL;
7152 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7153 bool do_bailout = 0;
7154
7155 zend_shared_alloc_lock();
7156
7157 /* Checks under lock */
7158 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7159 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7160 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7161 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7162 } else {
7163 zend_try {
7164 SHM_UNPROTECT();
7165 zend_jit_unprotect();
7166
7167 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7168
7169 t->id = ZEND_JIT_TRACE_NUM;
7170 t->root = ZEND_JIT_TRACE_NUM;
7171 t->parent = 0;
7172 t->link = 0;
7173 t->exit_count = 0;
7174 t->child_count = 0;
7175 t->stack_map_size = 0;
7176 t->flags = 0;
7177 t->polymorphism = 0;
7178 t->jmp_table_size = 0;
7179 t->op_array = trace_buffer[0].op_array;
7180 t->opline = trace_buffer[1].opline;
7181 t->exit_info = exit_info;
7182 t->stack_map = NULL;
7183
7184 orig_trigger = JIT_G(trigger);
7185 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7186
7187 handler = zend_jit_trace(trace_buffer, 0, 0);
7188
7189 JIT_G(trigger) = orig_trigger;
7190
7191 if (handler) {
7192 zend_jit_trace_exit_info *shared_exit_info = NULL;
7193
7194 t->exit_info = NULL;
7195 if (t->exit_count) {
7196 /* reallocate exit_info into shared memory */
7197 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7198 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7199
7200 if (!shared_exit_info) {
7201 if (t->stack_map) {
7202 efree(t->stack_map);
7203 t->stack_map = NULL;
7204 }
7205 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7206 goto exit;
7207 }
7208 memcpy(shared_exit_info, exit_info,
7209 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7210 t->exit_info = shared_exit_info;
7211 }
7212
7213 if (t->stack_map_size) {
7214 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7215 if (!shared_stack_map) {
7216 efree(t->stack_map);
7217 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7218 goto exit;
7219 }
7220 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7221 efree(t->stack_map);
7222 t->stack_map = shared_stack_map;
7223 }
7224
7225 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7226 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7227
7228 ((zend_op*)opline)->handler = handler;
7229
7230 ZEND_JIT_TRACE_NUM++;
7231 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7232
7233 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7234 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7235 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7236 if (t->stack_map) {
7237 efree(t->stack_map);
7238 t->stack_map = NULL;
7239 }
7240 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7241 } else {
7242 if (t->stack_map) {
7243 efree(t->stack_map);
7244 t->stack_map = NULL;
7245 }
7246 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7247 }
7248
7249 exit:;
7250 } zend_catch {
7251 do_bailout = 1;
7252 } zend_end_try();
7253
7254 zend_jit_protect();
7255 SHM_PROTECT();
7256 }
7257
7258 zend_shared_alloc_unlock();
7259
7260 if (do_bailout) {
7261 zend_bailout();
7262 }
7263
7264 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7265 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7266 && t->exit_count > 0) {
7267 zend_jit_dump_exit_info(t);
7268 }
7269
7270 return ret;
7271 }
7272
7273 /* Set counting handler back to original VM handler. */
zend_jit_stop_hot_trace_counters(zend_op_array * op_array)7274 static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array)
7275 {
7276 zend_jit_op_array_trace_extension *jit_extension;
7277 uint32_t i;
7278
7279 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7280 for (i = 0; i < op_array->last; i++) {
7281 /* Opline with Jit-ed code handler is skipped. */
7282 if (jit_extension->trace_info[i].trace_flags &
7283 (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7284 continue;
7285 }
7286 if (jit_extension->trace_info[i].trace_flags &
7287 (ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_START_RETURN)) {
7288 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7289 }
7290 }
7291 }
7292
7293 /* Get the tracing op_array. */
zend_jit_stop_persistent_op_array(zend_op_array * op_array)7294 static void zend_jit_stop_persistent_op_array(zend_op_array *op_array)
7295 {
7296 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
7297 if (!func_info) {
7298 return;
7299 }
7300 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
7301 zend_jit_stop_hot_trace_counters(op_array);
7302 }
7303 }
7304
7305 /* Get all op_arrays with counter handler. */
zend_jit_stop_persistent_script(zend_persistent_script * script)7306 static void zend_jit_stop_persistent_script(zend_persistent_script *script)
7307 {
7308 zend_class_entry *ce;
7309 zend_op_array *op_array;
7310
7311 zend_jit_stop_persistent_op_array(&script->script.main_op_array);
7312
7313 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
7314 zend_jit_stop_persistent_op_array(op_array);
7315 } ZEND_HASH_FOREACH_END();
7316
7317 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
7318 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
7319 if (op_array->type == ZEND_USER_FUNCTION) {
7320 zend_jit_stop_persistent_op_array(op_array);
7321 }
7322 } ZEND_HASH_FOREACH_END();
7323 } ZEND_HASH_FOREACH_END();
7324 }
7325
7326 /* Get all scripts which are accelerated by JIT */
zend_jit_stop_counter_handlers(void)7327 static void zend_jit_stop_counter_handlers(void)
7328 {
7329 if (ZCSG(jit_counters_stopped)) {
7330 return;
7331 }
7332
7333 zend_shared_alloc_lock();
7334 /* mprotect has an extreme overhead, avoid calls to it for every function. */
7335 SHM_UNPROTECT();
7336 if (!ZCSG(jit_counters_stopped)) {
7337 ZCSG(jit_counters_stopped) = true;
7338 for (uint32_t i = 0; i < ZCSG(hash).max_num_entries; i++) {
7339 zend_accel_hash_entry *cache_entry;
7340 for (cache_entry = ZCSG(hash).hash_table[i]; cache_entry; cache_entry = cache_entry->next) {
7341 zend_persistent_script *script;
7342 if (cache_entry->indirect) continue;
7343 script = (zend_persistent_script *)cache_entry->data;
7344 zend_jit_stop_persistent_script(script);
7345 }
7346 }
7347 }
7348 SHM_PROTECT();
7349 zend_shared_alloc_unlock();
7350 }
7351
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7352 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7353 {
7354 zend_shared_alloc_lock();
7355
7356 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7357 SHM_UNPROTECT();
7358 zend_jit_unprotect();
7359
7360 ((zend_op*)opline)->handler =
7361 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7362
7363 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7364
7365 zend_jit_protect();
7366 SHM_PROTECT();
7367 }
7368
7369 zend_shared_alloc_unlock();
7370 }
7371
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7372 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7373 {
7374 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7375 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7376 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7377 uint32_t cache_slot = JIT_G(bad_root_slot);
7378 uint32_t i;
7379
7380 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7381 if (cache_opline[i] == opline) {
7382 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7383 cache_opline[i] = NULL;
7384 return 1;
7385 } else {
7386 #if 0
7387 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7388 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7389 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7390 }
7391 #endif
7392 cache_count[i]++;
7393 cache_stop[i] = stop;
7394 return 0;
7395 }
7396 }
7397 }
7398 i = cache_slot;
7399 cache_opline[i] = opline;
7400 cache_count[i] = 1;
7401 cache_stop[i] = stop;
7402 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7403 JIT_G(bad_root_slot) = cache_slot;
7404 return 0;
7405 }
7406
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7407 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7408 {
7409 zend_jit_trace_rec *p = trace_buffer;
7410 const zend_op_array *op_array;
7411 const zend_op *opline;
7412 uint32_t level = 1 + trace_buffer[0].level;
7413 int idx, len, i, v, vars_count, call_level;
7414
7415 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7416 op_array = p->op_array;
7417 p += ZEND_JIT_TRACE_START_REC_SIZE;
7418 idx = 0;
7419 call_level = 0;
7420
7421 if (tssa && tssa->var_info) {
7422 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7423 vars_count = op_array->last_var;
7424 } else {
7425 vars_count = op_array->last_var + op_array->T;
7426 }
7427 for (i = 0; i < vars_count; i++) {
7428 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7429 fprintf(stderr, " %*c;", level, ' ');
7430 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7431 fprintf(stderr, "\n");
7432 }
7433 }
7434 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7435 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7436 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7437 zend_ssa_phi *p = tssa->blocks[1].phis;
7438
7439 fprintf(stderr, "LOOP:\n");
7440
7441 while (p) {
7442 fprintf(stderr, " ;");
7443 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7444 fprintf(stderr, " = Phi(");
7445 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7446 fprintf(stderr, ", ");
7447 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7448 fprintf(stderr, ")\n");
7449 p = p->next;
7450 }
7451 }
7452 }
7453
7454 while (1) {
7455 if (p->op == ZEND_JIT_TRACE_VM) {
7456 uint8_t op1_type, op2_type, op3_type;
7457
7458 opline = p->opline;
7459 fprintf(stderr, "%04d%*c",
7460 (int)(opline - op_array->opcodes),
7461 level, ' ');
7462 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7463
7464 op1_type = p->op1_type;
7465 op2_type = p->op2_type;
7466 op3_type = p->op3_type;
7467 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7468 fprintf(stderr, " ;");
7469 if (op1_type != IS_UNKNOWN) {
7470 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7471 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7472 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7473 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7474 p++;
7475 fprintf(stderr, " op1(%sobject of class %s)", ref,
7476 ZSTR_VAL(p->ce->name));
7477 } else {
7478 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7479 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7480 }
7481 }
7482 if (op2_type != IS_UNKNOWN) {
7483 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7484 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7485 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7486 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7487 p++;
7488 fprintf(stderr, " op2(%sobject of class %s)", ref,
7489 ZSTR_VAL(p->ce->name));
7490 } else {
7491 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7492 fprintf(stderr, " op2(%s%s)", ref, type);
7493 }
7494 }
7495 if (op3_type != IS_UNKNOWN) {
7496 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7497 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7498 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7499 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7500 fprintf(stderr, " op3(%s%s)", ref, type);
7501 }
7502 }
7503 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7504 uint8_t val_type;
7505 const char *type;
7506
7507 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7508 fprintf(stderr, " ;");
7509 }
7510 p++;
7511 val_type = p->op1_type;
7512
7513 if (val_type == IS_UNDEF) {
7514 type = "undef";
7515 } else if (val_type == IS_REFERENCE) {
7516 type = "ref";
7517 } else {
7518 type = zend_get_type_by_const(val_type);
7519 }
7520 fprintf(stderr, " val(%s)", type);
7521 }
7522 fprintf(stderr, "\n");
7523 idx++;
7524
7525 len = zend_jit_trace_op_len(opline);
7526 while (len > 1) {
7527 opline++;
7528 fprintf(stderr, "%04d%*c;",
7529 (int)(opline - op_array->opcodes),
7530 level, ' ');
7531 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7532 idx++;
7533 len--;
7534 fprintf(stderr, "\n");
7535 }
7536 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7537 op_array = p->op_array;
7538 fprintf(stderr, " %*c>enter %s%s%s\n",
7539 level, ' ',
7540 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7541 op_array->scope ? "::" : "",
7542 op_array->function_name ?
7543 ZSTR_VAL(op_array->function_name) :
7544 ZSTR_VAL(op_array->filename));
7545 level++;
7546 if (tssa && tssa->var_info) {
7547 call_level++;
7548 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7549 vars_count = op_array->last_var;
7550 for (i = 0; i < vars_count; i++, v++) {
7551 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7552 fprintf(stderr, " %*c;", level, ' ');
7553 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7554 fprintf(stderr, "\n");
7555 }
7556 }
7557 }
7558 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7559 op_array = p->op_array;
7560 level--;
7561 fprintf(stderr, " %*c<back %s%s%s\n",
7562 level, ' ',
7563 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7564 op_array->scope ? "::" : "",
7565 op_array->function_name ?
7566 ZSTR_VAL(op_array->function_name) :
7567 ZSTR_VAL(op_array->filename));
7568 if (tssa && tssa->var_info) {
7569 if (call_level == 0) {
7570 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7571 vars_count = op_array->last_var + op_array->T;
7572 for (i = 0; i < vars_count; i++, v++) {
7573 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7574 fprintf(stderr, " %*c;", level, ' ');
7575 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7576 fprintf(stderr, "\n");
7577 }
7578 }
7579 } else {
7580 call_level--;
7581 }
7582 }
7583 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7584 if (p->func != (zend_function*)&zend_pass_function) {
7585 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7586 level, ' ',
7587 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7588 (p->func && p->func->common.scope) ? "::" : "",
7589 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7590 } else {
7591 fprintf(stderr, " %*c>skip\n",
7592 level, ' ');
7593 }
7594 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7595 if (p->func != (zend_function*)&zend_pass_function) {
7596 fprintf(stderr, " %*c>call %s%s%s\n",
7597 level, ' ',
7598 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7599 p->func->common.scope ? "::" : "",
7600 ZSTR_VAL(p->func->common.function_name));
7601 } else {
7602 fprintf(stderr, " %*c>skip\n",
7603 level, ' ');
7604 }
7605 } else if (p->op == ZEND_JIT_TRACE_END) {
7606 break;
7607 }
7608 p++;
7609 }
7610 }
7611
zend_jit_dump_exit_info(zend_jit_trace_info * t)7612 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7613 {
7614 int i, j;
7615
7616 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7617 for (i = 0; i < t->exit_count; i++) {
7618 const zend_op_array *op_array = t->exit_info[i].op_array;
7619 uint32_t stack_size = t->exit_info[i].stack_size;
7620 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset;
7621
7622 fprintf(stderr, " exit_%d:", i);
7623 if (t->exit_info[i].opline) {
7624 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7625 } else {
7626 fprintf(stderr, " ----/");
7627 }
7628 if (t->exit_info[i].stack_size) {
7629 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7630 } else {
7631 fprintf(stderr, "----/0");
7632 }
7633 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7634 fprintf(stderr, "/VM");
7635 }
7636 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7637 fprintf(stderr, "/CALL");
7638 }
7639 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7640 fprintf(stderr, "/POLY");
7641 }
7642 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7643 fprintf(stderr, "/FREE_OP1");
7644 }
7645 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7646 fprintf(stderr, "/FREE_OP2");
7647 }
7648 for (j = 0; j < stack_size; j++) {
7649 uint8_t type = STACK_TYPE(stack, j);
7650 if (type != IS_UNKNOWN) {
7651 fprintf(stderr, " ");
7652 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7653 fprintf(stderr, ":");
7654 if (type == IS_UNDEF) {
7655 fprintf(stderr, "undef");
7656 } else {
7657 fprintf(stderr, "%s", zend_get_type_by_const(type));
7658 }
7659 if (STACK_REG(stack, j) != ZREG_NONE) {
7660 if (STACK_REG(stack, j) < ZREG_NUM) {
7661 fprintf(stderr, "(%s)", zend_reg_name[STACK_REG(stack, j)]);
7662 } else if (STACK_REG(stack, j) == ZREG_THIS) {
7663 fprintf(stderr, "(this)");
7664 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7665 fprintf(stderr, "(zval_try_addref)");
7666 } else {
7667 fprintf(stderr, "(const_%d)", STACK_REG(stack, j) - ZREG_NUM);
7668 }
7669 }
7670 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7671 fprintf(stderr, " ");
7672 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7673 fprintf(stderr, ":unknown(zval_try_addref)");
7674 } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_GPR0) {
7675 fprintf(stderr, " ");
7676 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7677 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[ZREG_COPY]);
7678 }
7679 }
7680 fprintf(stderr, "\n");
7681 }
7682 }
7683
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)7684 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
7685 {
7686 const zend_op *orig_opline;
7687 zend_jit_trace_stop stop;
7688 int ret = 0;
7689 zend_op_array *op_array;
7690 zend_jit_op_array_trace_extension *jit_extension;
7691 size_t offset;
7692 uint32_t trace_num;
7693 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7694
7695 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
7696 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
7697 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
7698
7699 repeat:
7700 trace_num = ZEND_JIT_TRACE_NUM;
7701 orig_opline = opline;
7702 op_array = &EX(func)->op_array;
7703 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7704 offset = jit_extension->offset;
7705
7706 EX(opline) = opline;
7707
7708 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
7709 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7710 return 0;
7711 }
7712
7713 if (JIT_G(tracing)) {
7714 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
7715 return 0;
7716 }
7717
7718 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7719 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7720 trace_num,
7721 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7722 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7723 EX(func)->op_array.scope ? "::" : "",
7724 EX(func)->op_array.function_name ?
7725 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7726 ZSTR_VAL(EX(func)->op_array.filename),
7727 opline->lineno);
7728 }
7729
7730 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7731 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7732 zend_jit_stop_counter_handlers();
7733 goto abort;
7734 }
7735
7736 JIT_G(tracing) = 1;
7737 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
7738 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
7739 JIT_G(tracing) = 0;
7740
7741 if (stop & ZEND_JIT_TRACE_HALT) {
7742 ret = -1;
7743 }
7744 stop &= ~ZEND_JIT_TRACE_HALT;
7745
7746 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
7747 orig_opline = trace_buffer[1].opline;
7748 op_array = (zend_op_array*)trace_buffer[0].op_array;
7749 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7750 offset = jit_extension->offset;
7751 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7752 const zend_op_array *op_array = trace_buffer[0].op_array;
7753 const zend_op *opline = trace_buffer[1].opline;
7754 zend_jit_op_array_trace_extension *jit_extension =
7755 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7756 size_t offset = jit_extension->offset;
7757
7758 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7759 trace_num,
7760 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7761 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7762 op_array->scope ? "::" : "",
7763 op_array->function_name ?
7764 ZSTR_VAL(op_array->function_name) : "$main",
7765 ZSTR_VAL(op_array->filename),
7766 opline->lineno);
7767 }
7768 }
7769
7770 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
7771 zend_jit_dump_trace(trace_buffer, NULL);
7772 }
7773
7774 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
7775 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
7776 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
7777 uint32_t idx = trace_buffer[1].last;
7778 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
7779 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
7780 trace_num,
7781 link_to);
7782 } else {
7783 fprintf(stderr, "---- TRACE %d stop (%s)\n",
7784 trace_num,
7785 zend_jit_trace_stop_description[stop]);
7786 }
7787 }
7788 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
7789 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
7790 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
7791 fprintf(stderr, "---- TRACE %d %s\n",
7792 trace_num,
7793 zend_jit_trace_stop_description[stop]);
7794 }
7795 } else {
7796 goto abort;
7797 }
7798 } else {
7799 abort:
7800 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
7801 fprintf(stderr, "---- TRACE %d abort (%s)\n",
7802 trace_num,
7803 zend_jit_trace_stop_description[stop]);
7804 }
7805 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
7806 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
7807 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7808 fprintf(stderr, "---- TRACE %d blacklisted\n",
7809 trace_num);
7810 }
7811 zend_jit_blacklist_root_trace(orig_opline, offset);
7812 }
7813 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
7814 execute_data = EG(current_execute_data);
7815 opline = EX(opline);
7816 goto repeat;
7817 }
7818 }
7819
7820 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
7821 fprintf(stderr, "\n");
7822 }
7823
7824 return ret;
7825 }
7826
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)7827 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
7828 {
7829 const void *handler;
7830 bool do_bailout = 0;
7831
7832 zend_shared_alloc_lock();
7833
7834 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
7835 SHM_UNPROTECT();
7836 zend_jit_unprotect();
7837
7838 zend_try {
7839 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
7840
7841 if (handler) {
7842 zend_jit_link_side_trace(
7843 zend_jit_traces[trace_num].code_start,
7844 zend_jit_traces[trace_num].code_size,
7845 zend_jit_traces[trace_num].jmp_table_size,
7846 exit_num,
7847 handler);
7848 }
7849 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
7850 } zend_catch {
7851 do_bailout = 1;
7852 } zend_end_try();
7853
7854 zend_jit_protect();
7855 SHM_PROTECT();
7856 }
7857
7858 zend_shared_alloc_unlock();
7859
7860 if (do_bailout) {
7861 zend_bailout();
7862 }
7863 }
7864
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)7865 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
7866 {
7867 uint8_t *counter = JIT_G(exit_counters) +
7868 zend_jit_traces[trace_num].exit_counters + exit_num;
7869
7870 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
7871 return 1;
7872 }
7873 (*counter)++;
7874 return 0;
7875 }
7876
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)7877 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
7878 {
7879 uint8_t *counter = JIT_G(exit_counters) +
7880 zend_jit_traces[trace_num].exit_counters + exit_num;
7881
7882 if (*counter + 1 >= JIT_G(hot_side_exit)) {
7883 return 1;
7884 }
7885 (*counter)++;
7886 return 0;
7887 }
7888
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)7889 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
7890 {
7891 zend_jit_trace_stop ret;
7892 const void *handler;
7893 uint8_t orig_trigger;
7894 zend_jit_trace_info *t;
7895 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7896 bool do_bailout = 0;
7897
7898 zend_shared_alloc_lock();
7899
7900 /* Checks under lock */
7901 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7902 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7903 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7904 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7905 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
7906 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
7907 } else {
7908 SHM_UNPROTECT();
7909 zend_jit_unprotect();
7910
7911 zend_try {
7912 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7913
7914 t->id = ZEND_JIT_TRACE_NUM;
7915 t->root = zend_jit_traces[parent_num].root;
7916 t->parent = parent_num;
7917 t->link = 0;
7918 t->exit_count = 0;
7919 t->child_count = 0;
7920 t->stack_map_size = 0;
7921 t->flags = 0;
7922 t->polymorphism = polymorphism;
7923 t->jmp_table_size = 0;
7924 t->opline = NULL;
7925 t->exit_info = exit_info;
7926 t->stack_map = NULL;
7927
7928 orig_trigger = JIT_G(trigger);
7929 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7930
7931 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
7932
7933 JIT_G(trigger) = orig_trigger;
7934
7935 if (handler) {
7936 zend_jit_trace_exit_info *shared_exit_info = NULL;
7937
7938 t->exit_info = NULL;
7939 if (t->exit_count) {
7940 /* reallocate exit_info into shared memory */
7941 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7942 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7943
7944 if (!shared_exit_info) {
7945 if (t->stack_map) {
7946 efree(t->stack_map);
7947 t->stack_map = NULL;
7948 }
7949 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7950 goto exit;
7951 }
7952 memcpy(shared_exit_info, exit_info,
7953 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7954 t->exit_info = shared_exit_info;
7955 }
7956
7957 if (t->stack_map_size) {
7958 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7959 if (!shared_stack_map) {
7960 efree(t->stack_map);
7961 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7962 goto exit;
7963 }
7964 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7965 efree(t->stack_map);
7966 t->stack_map = shared_stack_map;
7967 }
7968
7969 zend_jit_link_side_trace(
7970 zend_jit_traces[parent_num].code_start,
7971 zend_jit_traces[parent_num].code_size,
7972 zend_jit_traces[parent_num].jmp_table_size,
7973 exit_num,
7974 handler);
7975
7976 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7977 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7978
7979 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
7980 ZEND_JIT_TRACE_NUM++;
7981 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
7982
7983 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7984 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7985 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7986 if (t->stack_map) {
7987 efree(t->stack_map);
7988 t->stack_map = NULL;
7989 }
7990 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7991 } else {
7992 if (t->stack_map) {
7993 efree(t->stack_map);
7994 t->stack_map = NULL;
7995 }
7996 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7997 }
7998
7999 exit:;
8000 } zend_catch {
8001 do_bailout = 1;
8002 } zend_end_try();
8003
8004 zend_jit_protect();
8005 SHM_PROTECT();
8006 }
8007
8008 zend_shared_alloc_unlock();
8009
8010 if (do_bailout) {
8011 zend_bailout();
8012 }
8013
8014 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
8015 && ret == ZEND_JIT_TRACE_STOP_COMPILED
8016 && t->exit_count > 0) {
8017 zend_jit_dump_exit_info(t);
8018 }
8019
8020 return ret;
8021 }
8022
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)8023 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
8024 {
8025 zend_jit_trace_stop stop;
8026 int ret = 0;
8027 uint32_t trace_num;
8028 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8029 uint32_t is_megamorphic = 0;
8030 uint32_t polymorphism = 0;
8031
8032 trace_num = ZEND_JIT_TRACE_NUM;
8033
8034 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8035 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8036 return 0;
8037 }
8038
8039 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8040 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
8041 trace_num, parent_num, exit_num,
8042 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8043 EX(func)->op_array.scope ? "::" : "",
8044 EX(func)->op_array.function_name ?
8045 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8046 ZSTR_VAL(EX(func)->op_array.filename),
8047 EX(opline)->lineno);
8048 }
8049
8050 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8051 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8052 goto abort;
8053 }
8054
8055 if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8056 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8057 goto abort;
8058 }
8059
8060 if (JIT_G(max_polymorphic_calls) > 0) {
8061 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
8062 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
8063 && EX(call))) {
8064 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
8065 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
8066 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
8067 } else if (!zend_jit_traces[parent_num].polymorphism) {
8068 polymorphism = 1;
8069 } else if (exit_num == 0) {
8070 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
8071 }
8072 }
8073 }
8074
8075 JIT_G(tracing) = 1;
8076 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
8077 JIT_G(tracing) = 0;
8078
8079 if (stop & ZEND_JIT_TRACE_HALT) {
8080 ret = -1;
8081 }
8082 stop &= ~ZEND_JIT_TRACE_HALT;
8083
8084 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8085 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8086 const zend_op_array *op_array = trace_buffer[0].op_array;
8087 const zend_op *opline = trace_buffer[1].opline;
8088 zend_jit_op_array_trace_extension *jit_extension =
8089 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8090 size_t offset = jit_extension->offset;
8091
8092 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8093 trace_num,
8094 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8095 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8096 op_array->scope ? "::" : "",
8097 op_array->function_name ?
8098 ZSTR_VAL(op_array->function_name) : "$main",
8099 ZSTR_VAL(op_array->filename),
8100 opline->lineno);
8101 }
8102 }
8103
8104 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8105 zend_jit_dump_trace(trace_buffer, NULL);
8106 }
8107
8108 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8109 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8110 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8111 uint32_t idx = trace_buffer[1].last;
8112 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8113 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8114 trace_num,
8115 link_to);
8116 } else {
8117 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8118 trace_num,
8119 zend_jit_trace_stop_description[stop]);
8120 }
8121 }
8122 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8123 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8124 } else {
8125 const zend_op_array *op_array = trace_buffer[0].op_array;
8126 zend_jit_op_array_trace_extension *jit_extension =
8127 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8128 const zend_op *opline = trace_buffer[1].opline;
8129
8130 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8131 }
8132 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8133 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8134 fprintf(stderr, "---- TRACE %d %s\n",
8135 trace_num,
8136 zend_jit_trace_stop_description[stop]);
8137 }
8138 } else {
8139 goto abort;
8140 }
8141 } else {
8142 abort:
8143 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8144 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8145 trace_num,
8146 zend_jit_trace_stop_description[stop]);
8147 }
8148 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8149 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8150 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8151 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8152 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8153 parent_num, exit_num);
8154 }
8155 }
8156 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8157 execute_data = EG(current_execute_data);
8158 return zend_jit_trace_hot_root(execute_data, EX(opline));
8159 }
8160 }
8161
8162 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8163 fprintf(stderr, "\n");
8164 }
8165
8166 return ret;
8167 }
8168
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8169 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8170 {
8171 uint32_t trace_num = EG(jit_trace_num);
8172 zend_execute_data *execute_data = EG(current_execute_data);
8173 const zend_op *orig_opline = EX(opline);
8174 const zend_op *opline;
8175 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8176 int repeat_last_opline = 0;
8177
8178 /* Deoptimization of VM stack state */
8179 uint32_t i;
8180 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8181 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset;
8182
8183 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8184 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8185 call->prev_execute_data = EX(call);
8186 EX(call) = call;
8187 }
8188
8189 for (i = 0; i < stack_size; i++) {
8190 if (STACK_REG(stack, i) != ZREG_NONE) {
8191 if (STACK_TYPE(stack, i) == IS_LONG) {
8192 zend_long val;
8193
8194 if (STACK_REG(stack, i) < ZREG_NUM) {
8195 val = regs->gpr[STACK_REG(stack, i)];
8196 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN) {
8197 val = ZEND_LONG_MIN;
8198 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX) {
8199 val = ZEND_LONG_MAX;
8200 } else {
8201 ZEND_UNREACHABLE();
8202 }
8203 ZVAL_LONG(EX_VAR_NUM(i), val);
8204 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8205 double val;
8206
8207 if (STACK_REG(stack, i) < ZREG_NUM) {
8208 val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8209 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN_MINUS_1) {
8210 val = (double)ZEND_LONG_MIN - 1.0;
8211 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX_PLUS_1) {
8212 val = (double)ZEND_LONG_MAX + 1.0;
8213 } else {
8214 ZEND_UNREACHABLE();
8215 }
8216 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8217 } else if (STACK_REG(stack, i) == ZREG_THIS) {
8218 zend_object *obj = Z_OBJ(EX(This));
8219
8220 GC_ADDREF(obj);
8221 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8222 } else if (STACK_REG(stack, i) == ZREG_NULL) {
8223 ZVAL_NULL(EX_VAR_NUM(i));
8224 } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) {
8225 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8226 } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_GPR0) {
8227 zval *val = (zval*)regs->gpr[ZREG_COPY];
8228
8229 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8230 /* Undefined array index or property */
8231 repeat_last_opline = 1;
8232 } else {
8233 ZVAL_COPY(EX_VAR_NUM(i), val);
8234 }
8235 } else {
8236 ZEND_UNREACHABLE();
8237 }
8238 }
8239 }
8240
8241 if (repeat_last_opline) {
8242 EX(opline) = t->exit_info[exit_num].opline - 1;
8243 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8244 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8245 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8246 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8247 }
8248 return 1;
8249 }
8250
8251 opline = t->exit_info[exit_num].opline;
8252
8253 if (opline) {
8254 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8255 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8256 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8257 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8258 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8259 EX(opline) = opline-1;
8260 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8261 }
8262 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8263 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8264 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8265 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8266 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8267 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8268 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8269 EX(opline) = opline-1;
8270 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8271 }
8272 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8273 if (EG(exception)) {
8274 return 1;
8275 }
8276 }
8277 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8278 zend_function *func = (zend_function*)regs->gpr[ZREG_COPY];
8279
8280 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8281 zend_string_release_ex(func->common.function_name, 0);
8282 zend_free_trampoline(func);
8283 EX(opline) = opline;
8284 return 1;
8285 }
8286 }
8287
8288 /* Set VM opline to continue interpretation */
8289 EX(opline) = opline;
8290 }
8291
8292 if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) {
8293 return 1;
8294 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8295 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8296 return 0;
8297 }
8298
8299 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8300 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8301 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8302
8303 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8304 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8305 trace_num,
8306 exit_num,
8307 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8308 EX(func)->op_array.scope ? "::" : "",
8309 EX(func)->op_array.function_name ?
8310 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8311 ZSTR_VAL(EX(func)->op_array.filename),
8312 EX(opline)->lineno);
8313 }
8314
8315 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8316 zend_jit_op_array_trace_extension *jit_extension;
8317 uint32_t num = trace_num;
8318
8319 while (t->root != num) {
8320 num = t->root;
8321 t = &zend_jit_traces[num];
8322 }
8323
8324 zend_shared_alloc_lock();
8325
8326 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8327
8328 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8329 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8330 /* skip: not JIT-ed nor blacklisted */
8331 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8332 /* too many root traces, blacklist the root trace */
8333 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
8334 SHM_UNPROTECT();
8335 zend_jit_unprotect();
8336
8337 ((zend_op*)opline)->handler =
8338 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->orig_handler;
8339
8340 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= ~ZEND_JIT_TRACE_JITED;
8341 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
8342
8343 zend_jit_protect();
8344 SHM_PROTECT();
8345 }
8346 } else {
8347 SHM_UNPROTECT();
8348 zend_jit_unprotect();
8349
8350 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8351 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8352 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8353 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8354 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8355 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8356 }
8357 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8358 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8359
8360 zend_jit_protect();
8361 SHM_PROTECT();
8362 }
8363
8364 zend_shared_alloc_unlock();
8365
8366 return 0;
8367 }
8368
8369 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8370 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8371 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8372 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8373 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8374 trace_num, exit_num);
8375 }
8376 return 0;
8377 }
8378 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8379 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8380 }
8381
8382 /* Return 1 to call original handler instead of the same JIT-ed trace */
8383 return (orig_opline == t->opline && EX(opline) == orig_opline);
8384 }
8385
zend_jit_trace_supported(const zend_op * opline)8386 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8387 {
8388 switch (opline->opcode) {
8389 case ZEND_CATCH:
8390 case ZEND_FAST_CALL:
8391 case ZEND_FAST_RET:
8392 return ZEND_JIT_TRACE_UNSUPPORTED;
8393 default:
8394 return ZEND_JIT_TRACE_SUPPORTED;
8395 }
8396 }
8397
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8398 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8399 {
8400 zend_jit_op_array_trace_extension *jit_extension;
8401 uint32_t i;
8402
8403 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8404 for (i = 0; i < op_array->last; i++) {
8405 jit_extension->trace_info[i].trace_flags &=
8406 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8407 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8408 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8409 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8410 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8411 } else {
8412 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8413 }
8414 }
8415 return SUCCESS;
8416 }
8417
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8418 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8419 {
8420 zend_op *opline;
8421 zend_jit_op_array_trace_extension *jit_extension;
8422 uint32_t i;
8423
8424 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8425
8426 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8427 if (!jit_extension) {
8428 return FAILURE;
8429 }
8430 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8431 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8432 jit_extension->op_array = op_array;
8433 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8434 for (i = 0; i < op_array->last; i++) {
8435 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8436 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8437 jit_extension->trace_info[i].counter = NULL;
8438 jit_extension->trace_info[i].trace_flags =
8439 zend_jit_trace_supported(&op_array->opcodes[i]);
8440 }
8441 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8442
8443 if (JIT_G(hot_loop)) {
8444 zend_cfg cfg;
8445
8446 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8447
8448 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8449 return FAILURE;
8450 }
8451
8452 for (i = 0; i < cfg.blocks_count; i++) {
8453 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8454 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8455 /* loop header */
8456 opline = op_array->opcodes + cfg.blocks[i].start;
8457 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8458 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8459 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8460 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8461 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8462 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8463 }
8464 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8465 ZEND_JIT_TRACE_START_LOOP;
8466 }
8467 }
8468 }
8469 }
8470 }
8471
8472 if (JIT_G(hot_func)) {
8473 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8474 opline = op_array->opcodes;
8475 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8476 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8477 opline++;
8478 }
8479 }
8480
8481 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8482 /* function entry */
8483 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8484 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8485 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8486 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8487 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8488 ZEND_JIT_TRACE_START_ENTER;
8489 }
8490 }
8491
8492 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8493
8494 return SUCCESS;
8495 }
8496
zend_jit_trace_init_caches(void)8497 static void zend_jit_trace_init_caches(void)
8498 {
8499 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8500 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8501 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8502 JIT_G(bad_root_slot) = 0;
8503
8504 if (JIT_G(exit_counters)) {
8505 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8506 }
8507 }
8508
zend_jit_trace_reset_caches(void)8509 static void zend_jit_trace_reset_caches(void)
8510 {
8511 JIT_G(tracing) = 0;
8512 #ifdef ZTS
8513 if (!JIT_G(exit_counters)) {
8514 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8515 }
8516 #endif
8517 }
8518
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8519 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8520 {
8521 if (jit_globals->exit_counters) {
8522 free(jit_globals->exit_counters);
8523 }
8524 }
8525
zend_jit_trace_restart(void)8526 static void zend_jit_trace_restart(void)
8527 {
8528 ZEND_JIT_TRACE_NUM = 1;
8529 ZEND_JIT_COUNTER_NUM = 0;
8530 ZEND_JIT_EXIT_NUM = 0;
8531 ZEND_JIT_EXIT_COUNTERS = 0;
8532 ZCSG(jit_counters_stopped) = false;
8533
8534 zend_jit_trace_init_caches();
8535 }
8536
