1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(bool reattached)49 static int zend_jit_trace_startup(bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 return FAILURE;
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 return FAILURE;
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 ZCSG(jit_counters_stopped) = false;
67 } else {
68 zend_jit_traces = ZCSG(jit_traces);
69 if (!zend_jit_traces) {
70 return FAILURE;
71 }
72 zend_jit_exit_groups = ZCSG(jit_exit_groups);
73 if (!zend_jit_exit_groups) {
74 return FAILURE;
75 }
76 }
77
78 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
79 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
80
81 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
82 if (JIT_G(exit_counters) == NULL) {
83 return FAILURE;
84 }
85
86 return SUCCESS;
87 }
88
zend_jit_trace_allocate_exit_group(uint32_t n)89 static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
90 {
91 dasm_State* dasm_state = NULL;
92 const void *entry;
93 char name[32];
94
95 dasm_init(&dasm_state, DASM_MAXSECTION);
96 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
97 dasm_setup(&dasm_state, dasm_actions);
98 zend_jit_align_stub(&dasm_state);
99 zend_jit_trace_exit_group_stub(&dasm_state, n);
100
101 sprintf(name, "jit$$trace_exit_%d", n);
102 entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE);
103 dasm_free(&dasm_state);
104
105 #ifdef HAVE_DISASM
106 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
107 uint32_t i;
108
109 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
110 sprintf(name, "jit$$trace_exit_%d", n + i);
111 zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
112 }
113 }
114 #endif
115
116 return entry;
117 }
118
zend_jit_trace_allocate_exit_point(uint32_t n)119 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
120 {
121 const void *group = NULL;
122
123 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
124 return NULL;
125 }
126 do {
127 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
128 if (!group) {
129 return NULL;
130 }
131 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
132 group;
133 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
134 } while (n >= ZEND_JIT_EXIT_NUM);
135 return (const void*)
136 ((const char*)group +
137 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
138 }
139
zend_jit_trace_get_exit_addr(uint32_t n)140 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
141 {
142 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
143 return zend_jit_trace_allocate_exit_point(n);
144 }
145 return (const void*)
146 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
147 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
148 }
149
150 #if ZEND_JIT_TARGET_ARM64
zend_jit_get_current_trace_info(void)151 static zend_jit_trace_info *zend_jit_get_current_trace_info(void)
152 {
153 return &zend_jit_traces[ZEND_JIT_TRACE_NUM];
154 }
155
zend_jit_trace_find_exit_point(const void * addr)156 static uint32_t zend_jit_trace_find_exit_point(const void* addr)
157 {
158 uint32_t n = ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP;
159 uint32_t i;
160
161 for (i = 0; i < n; i++) {
162 if ((const char*)addr >= (const char*)zend_jit_exit_groups[i]
163 && (const char*)addr < (const char*)zend_jit_exit_groups[i] +
164 (ZEND_JIT_EXIT_POINTS_PER_GROUP * ZEND_JIT_EXIT_POINTS_SPACING)) {
165 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
166 ((const char*)addr - (const char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING;
167 }
168 }
169 return (uint32_t)-1;
170 }
171 #endif
172
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)173 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
174 {
175 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
176 uint32_t exit_point;
177 const zend_op_array *op_array;
178 uint32_t stack_offset = (uint32_t)-1;
179 uint32_t stack_size;
180 zend_jit_trace_stack *stack = NULL;
181
182 if (delayed_call_chain) {
183 assert(to_opline != NULL); /* CALL and IP share the same register */
184 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
185 }
186 if (JIT_G(current_frame)) {
187 op_array = &JIT_G(current_frame)->func->op_array;
188 stack_size = op_array->last_var + op_array->T;
189 if (stack_size) {
190 stack = JIT_G(current_frame)->stack;
191 do {
192 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
193 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
194 || STACK_REG(stack, stack_size-1) != ZREG_NONE) {
195 break;
196 }
197 stack_size--;
198 } while (stack_size);
199 }
200 } else {
201 op_array = NULL;
202 stack_size = 0;
203 }
204
205 /* Try to reuse exit points */
206 if (to_opline != NULL && t->exit_count > 0) {
207 uint32_t i = t->exit_count;
208
209 do {
210 i--;
211 if (stack_size == 0
212 || (t->exit_info[i].stack_size >= stack_size
213 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
214 stack_offset = t->exit_info[i].stack_offset;
215 if (t->exit_info[i].opline == to_opline
216 && t->exit_info[i].flags == flags
217 && t->exit_info[i].stack_size == stack_size) {
218 return i;
219 }
220 }
221 } while (i > 0);
222 }
223
224 exit_point = t->exit_count;
225 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
226 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
227 stack_offset = t->stack_map_size;
228 t->stack_map_size += stack_size;
229 // TODO: reduce number of reallocations ???
230 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
231 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
232 }
233 t->exit_count++;
234 t->exit_info[exit_point].opline = to_opline;
235 t->exit_info[exit_point].op_array = op_array;
236 t->exit_info[exit_point].flags = flags;
237 t->exit_info[exit_point].stack_size = stack_size;
238 t->exit_info[exit_point].stack_offset = stack_offset;
239 }
240
241 return exit_point;
242 }
243
zend_jit_trace_add_code(const void * start,uint32_t size)244 static void zend_jit_trace_add_code(const void *start, uint32_t size)
245 {
246 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
247
248 t->code_start = start;
249 t->code_size = size;
250 }
251
252 /**
253 * Locate a trace in the #zend_jit_traces array with the specified
254 * #code_start address.
255 *
256 * @return the #zend_jit_traces index or 0 if no such #code_start
257 * address was found
258 */
zend_jit_find_trace(const void * addr)259 static uint32_t zend_jit_find_trace(const void *addr)
260 {
261 uint32_t i;
262
263 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
264 if (zend_jit_traces[i].code_start == addr) {
265 return i;
266 }
267 }
268 return 0;
269 }
270
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)271 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
272 {
273 smart_str buf = {0};
274
275 smart_str_appends(&buf, TRACE_PREFIX);
276 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
277 smart_str_appendc(&buf, '$');
278 if (op_array->function_name) {
279 if (op_array->scope) {
280 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
281 smart_str_appends(&buf, "::");
282 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
283 } else {
284 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
285 }
286 } else if (op_array->filename) {
287 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
288 }
289 smart_str_appendc(&buf, '$');
290 smart_str_append_long(&buf, (zend_long)lineno);
291 smart_str_0(&buf);
292 return buf.s;
293 }
294
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)295 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
296 {
297 switch (opline->opcode) {
298 case ZEND_IS_IDENTICAL:
299 case ZEND_IS_NOT_IDENTICAL:
300 case ZEND_IS_EQUAL:
301 case ZEND_IS_NOT_EQUAL:
302 case ZEND_IS_SMALLER:
303 case ZEND_IS_SMALLER_OR_EQUAL:
304 case ZEND_CASE:
305 case ZEND_CASE_STRICT:
306 case ZEND_ISSET_ISEMPTY_CV:
307 case ZEND_ISSET_ISEMPTY_VAR:
308 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
309 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
310 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
311 case ZEND_INSTANCEOF:
312 case ZEND_TYPE_CHECK:
313 case ZEND_DEFINED:
314 case ZEND_IN_ARRAY:
315 case ZEND_ARRAY_KEY_EXISTS:
316 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
317 /* smart branch */
318 return 1;
319 }
320 break;
321 case ZEND_JMPZ:
322 case ZEND_JMPNZ:
323 case ZEND_JMPZ_EX:
324 case ZEND_JMPNZ_EX:
325 case ZEND_JMP_SET:
326 case ZEND_COALESCE:
327 case ZEND_JMP_NULL:
328 case ZEND_FE_RESET_R:
329 case ZEND_FE_RESET_RW:
330 case ZEND_ASSERT_CHECK:
331 case ZEND_FE_FETCH_R:
332 case ZEND_FE_FETCH_RW:
333 case ZEND_SWITCH_LONG:
334 case ZEND_SWITCH_STRING:
335 case ZEND_MATCH:
336 case ZEND_BIND_INIT_STATIC_OR_JMP:
337 /* branch opcodes */
338 return 1;
339 case ZEND_NEW:
340 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
341 /* NEW may skip constructor without arguments */
342 return 1;
343 }
344 break;
345 case ZEND_CATCH:
346 case ZEND_FAST_CALL:
347 case ZEND_FAST_RET:
348 case ZEND_GENERATOR_CREATE:
349 case ZEND_GENERATOR_RETURN:
350 case ZEND_EXIT:
351 case ZEND_YIELD:
352 case ZEND_YIELD_FROM:
353 case ZEND_INCLUDE_OR_EVAL:
354 case ZEND_MATCH_ERROR:
355 /* unsupported */
356 return 1;
357 case ZEND_DO_FCALL:
358 /* potentially polymorphic call */
359 return 1;
360 #if 0
361 case ZEND_DO_UCALL:
362 case ZEND_DO_FCALL_BY_NAME:
363 /* monomorphic call */
364 // TODO: recompilation may change target ???
365 return 0;
366 #endif
367 case ZEND_RETURN_BY_REF:
368 case ZEND_RETURN:
369 /* return */
370 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
371 default:
372 break;
373 }
374 return 0;
375 }
376
zend_jit_trace_type_to_info_ex(uint8_t type,uint32_t info)377 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(uint8_t type, uint32_t info)
378 {
379 if (type == IS_UNKNOWN) {
380 return info;
381 }
382 ZEND_ASSERT(info & (1 << type));
383 if (type < IS_STRING) {
384 return (1 << type);
385 } else if (type != IS_ARRAY) {
386 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
387 } else {
388 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
389 }
390 }
391
zend_jit_trace_type_to_info(uint8_t type)392 static zend_always_inline uint32_t zend_jit_trace_type_to_info(uint8_t type)
393 {
394 return zend_jit_trace_type_to_info_ex(type, -1);
395 }
396
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)397 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
398 {
399 if (var >= op_array->last_var) {
400 return NO_ALIAS;
401 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
402 return SYMTABLE_ALIAS;
403 } else if (ssa->vars) {
404 return ssa->vars[var].alias;
405 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
406 return HTTP_RESPONSE_HEADER_ALIAS;
407 }
408 return NO_ALIAS;
409 }
410
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)411 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
412 int ssa_var,
413 uint8_t op_type)
414 {
415 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
416
417 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
418 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
419 info->type |= MAY_BE_GUARD;
420 } else {
421 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
422 }
423 }
424 }
425
426 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
427 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
428 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
429 } \
430 } while (0)
431
432 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
433 if (op_type != IS_UNKNOWN) { \
434 if ((op_info & MAY_BE_GUARD) != 0) { \
435 if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \
436 goto jit_failure; \
437 } \
438 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
439 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
440 op_info = zend_jit_trace_type_to_info(op_type); \
441 } else { \
442 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
443 op_info &= ~MAY_BE_GUARD; \
444 ssa->var_info[_ssa_var].type &= op_info; \
445 } \
446 } \
447 } \
448 } while (0)
449
450 #define ADD_OP1_TRACE_GUARD() \
451 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
452 #define ADD_OP2_TRACE_GUARD() \
453 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
454 #define ADD_OP1_DATA_TRACE_GUARD() \
455 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
456
457 #define CHECK_OP1_TRACE_TYPE() \
458 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
459 #define CHECK_OP2_TRACE_TYPE() \
460 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
461 #define CHECK_OP1_DATA_TRACE_TYPE() \
462 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
463
zend_jit_trace_frame_size(const zend_op_array * op_array)464 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
465 {
466 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
467 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
468 } else if (op_array) {
469 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
470 } else {
471 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
472 }
473 }
474
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)475 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
476 {
477 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
478 }
479
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)480 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
481 {
482 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
483 }
484
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,uint8_t type)485 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type)
486 {
487 zend_jit_trace_stack *stack = call->stack;
488 const zend_op_array *op_array = &call->func->op_array;
489 uint32_t arg_num = opline->op2.num;
490
491 if (arg_num > op_array->num_args) {
492 return;
493 }
494 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
495 zend_arg_info *arg_info;
496
497 ZEND_ASSERT(arg_num <= op_array->num_args);
498 arg_info = &op_array->arg_info[arg_num-1];
499
500 if (ZEND_TYPE_IS_SET(arg_info->type)) {
501 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
502 return;
503 }
504 }
505 }
506 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
507 }
508
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)509 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
510 {
511 if (func
512 && func->type == ZEND_INTERNAL_FUNCTION
513 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
514 && arg_num < func->internal_function.num_args) {
515 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
516
517 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
518 && ZEND_TYPE_IS_SET(arg_info->type)
519 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
520 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
521 && JIT_G(current_frame)
522 && JIT_G(current_frame)->call
523 && JIT_G(current_frame)->call->func) {
524 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
525
526 if (type != IS_UNKNOWN
527 && type < IS_STRING
528 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
529 return 0;
530 }
531 }
532 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
533 const zend_op *opline = call_info->arg_info[arg_num].opline;
534
535 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
536 zval *zv = RT_CONSTANT(opline, opline->op1);
537
538 if (!Z_REFCOUNTED_P(zv)) {
539 uint32_t type = Z_TYPE_P(zv);
540
541 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
542 if (type != IS_ARRAY
543 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
544 return 0;
545 }
546 }
547 }
548 }
549 }
550 }
551
552 return 1;
553 }
554
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)555 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
556 {
557 zend_jit_op_array_trace_extension *jit_extension;
558 zend_ssa *ssa;
559
560 jit_extension =
561 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
562 jit_extension->func_info.num = 0;
563 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
564 | ZEND_FUNC_JIT_ON_PROF_REQUEST
565 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
566 | ZEND_FUNC_JIT_ON_HOT_TRACE;
567 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
568 ssa = &jit_extension->func_info.ssa;
569
570 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
571 do {
572 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
573 break;
574 }
575
576 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
577 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info);
578 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
579 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
580 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
581 }
582 }
583
584 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
585 break;
586 }
587
588 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
589 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
590 }
591 return ssa;
592 } while (0);
593 }
594
595 memset(ssa, 0, sizeof(zend_ssa));
596 ssa->cfg.blocks_count = 1;
597
598 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
599 zend_cfg cfg;
600 void *checkpoint = zend_arena_checkpoint(CG(arena));
601
602 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
603 ssa->cfg.flags = cfg.flags;
604 } else{
605 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
606 }
607
608 /* TODO: move this to zend_cfg.c ? */
609 if (!op_array->function_name) {
610 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
611 }
612
613 zend_arena_release(&CG(arena), checkpoint);
614 }
615
616 return ssa;
617 }
618
619 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
620 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
621
zend_jit_trace_op_len(const zend_op * opline)622 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
623 {
624 int len;
625
626 switch (opline->opcode) {
627 case ZEND_ASSIGN_DIM:
628 case ZEND_ASSIGN_OBJ:
629 case ZEND_ASSIGN_STATIC_PROP:
630 case ZEND_ASSIGN_DIM_OP:
631 case ZEND_ASSIGN_OBJ_OP:
632 case ZEND_ASSIGN_STATIC_PROP_OP:
633 case ZEND_ASSIGN_OBJ_REF:
634 case ZEND_ASSIGN_STATIC_PROP_REF:
635 return 2; /* OP_DATA */
636 case ZEND_RECV_INIT:
637 len = 1;
638 opline++;
639 while (opline->opcode == ZEND_RECV_INIT) {
640 len++;
641 opline++;
642 }
643 return len;
644 case ZEND_BIND_GLOBAL:
645 len = 1;
646 opline++;
647 while (opline->opcode == ZEND_BIND_GLOBAL) {
648 len++;
649 opline++;
650 }
651 return len;
652 // case ZEND_IS_IDENTICAL:
653 // case ZEND_IS_NOT_IDENTICAL:
654 // case ZEND_IS_EQUAL:
655 // case ZEND_IS_NOT_EQUAL:
656 // case ZEND_IS_SMALLER:
657 // case ZEND_IS_SMALLER_OR_EQUAL:
658 // case ZEND_CASE:
659 // case ZEND_ISSET_ISEMPTY_CV:
660 // case ZEND_ISSET_ISEMPTY_VAR:
661 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
662 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
663 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
664 // case ZEND_INSTANCEOF:
665 // case ZEND_TYPE_CHECK:
666 // case ZEND_DEFINED:
667 // case ZEND_IN_ARRAY:
668 // case ZEND_ARRAY_KEY_EXISTS:
669 default:
670 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
671 return 2; /* JMPZ/JMPNZ */
672 }
673 return 1;
674 }
675 }
676
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)677 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
678 {
679 const zend_op_array *op_array;
680 zend_jit_trace_rec *p;
681 int k, vars_count;
682 zend_bitset use, def;
683 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
684 uint32_t set_size;
685 zend_ssa_phi *prev = NULL;
686 int level = 0;
687 ALLOCA_FLAG(use_heap);
688
689 op_array = trace_buffer->op_array;
690 set_size = zend_bitset_len(op_array->last_var + op_array->T);
691 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
692 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
693 def = use + set_size;
694 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
695 for (;;p++) {
696 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
697 const zend_op *opline = p->opline;
698 int len;
699
700 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
701 len = zend_jit_trace_op_len(opline);
702 while (len > 1) {
703 opline++;
704 if (opline->opcode != ZEND_OP_DATA) {
705 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
706 }
707 len--;
708 }
709 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
710 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
711 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
712 level++;
713 } else if (p->op == ZEND_JIT_TRACE_BACK) {
714 if (level == 0) {
715 // Phi for recursive calls and returns are not supported yet ???
716 assert(0);
717 } else {
718 level--;
719 }
720 } else if (p->op == ZEND_JIT_TRACE_END) {
721 break;
722 }
723 }
724
725 zend_bitset_intersection(use, def, set_size);
726
727 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
728 vars_count = op_array->last_var;
729 } else {
730 vars_count = op_array->last_var + op_array->T;
731 }
732 for (k = 0; k < vars_count; k++) {
733 if (zend_bitset_in(use, k)) {
734 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
735 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
736 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
737 sizeof(void*) * 2);
738 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
739 phi->sources[0] = STACK_VAR(stack, k);
740 phi->sources[1] = -1;
741 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
742 phi->pi = -1;
743 phi->var = k;
744 phi->ssa_var = ssa_vars_count;
745 SET_STACK_VAR(stack, k, ssa_vars_count);
746 ssa_vars_count++;
747 phi->block = 1;
748 if (prev) {
749 prev->next = phi;
750 } else {
751 tssa->blocks[1].phis = phi;
752 }
753 prev = phi;
754 }
755 }
756
757 free_alloca(use, use_heap);
758
759 return ssa_vars_count;
760 }
761
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)762 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
763 {
764 zend_ssa_phi *prev = NULL;
765 const zend_op_array *op_array = trace_buffer->op_array;
766 const zend_op *opline = trace_buffer[1].opline;
767 int count = opline - op_array->opcodes;
768 int i;
769
770 for(i = 0; i < count; i++) {
771 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
772 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
773 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
774 sizeof(void*) * 2);
775 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
776 phi->sources[0] = STACK_VAR(stack, i);
777 phi->sources[1] = -1;
778 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
779 phi->pi = -1;
780 phi->var = i;
781 phi->ssa_var = ssa_vars_count;
782 SET_STACK_VAR(stack, i, ssa_vars_count);
783 ssa_vars_count++;
784 phi->block = 1;
785 if (prev) {
786 prev->next = phi;
787 } else {
788 tssa->blocks[1].phis = phi;
789 }
790 prev = phi;
791 }
792 return ssa_vars_count;
793 }
794
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)795 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
796 {
797 const zend_op *opline = trace_buffer[1].opline - 1;
798 int i;
799
800 if (RETURN_VALUE_USED(opline)) {
801 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
802 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
803 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
804 sizeof(void*) * 2);
805
806 i = EX_VAR_TO_NUM(opline->result.var);
807 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
808 phi->sources[0] = STACK_VAR(stack, i);
809 phi->sources[1] = -1;
810 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
811 phi->pi = -1;
812 phi->var = i;
813 phi->ssa_var = ssa_vars_count;
814 SET_STACK_VAR(stack, i, ssa_vars_count);
815 ssa_vars_count++;
816 phi->block = 1;
817 tssa->blocks[1].phis = phi;
818 }
819 return ssa_vars_count;
820 }
821
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)822 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
823 {
824 int var, use;
825 zend_ssa_op *op;
826 zend_ssa_var_info *info;
827 unsigned int no_val;
828 zend_ssa_alias_kind alias;
829
830 if (tssa->vars[ssa_var].phi_use_chain) {
831 // TODO: this may be incorrect ???
832 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
833 } else {
834 var = ssa_var;
835 }
836 use = tssa->vars[var].use_chain;
837 if (use >= 0) {
838 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
839 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
840 if (tssa->ops[use].op1_use == var) {
841 no_val = ssa->vars[op->op1_use].no_val;
842 alias = ssa->vars[op->op1_use].alias;
843 info = ssa->var_info + op->op1_use;
844 } else if (tssa->ops[use].op2_use == var) {
845 no_val = ssa->vars[op->op2_use].no_val;
846 alias = ssa->vars[op->op2_use].alias;
847 info = ssa->var_info + op->op2_use;
848 } else if (tssa->ops[use].result_use == var) {
849 no_val = ssa->vars[op->result_use].no_val;
850 alias = ssa->vars[op->result_use].alias;
851 info = ssa->var_info + op->result_use;
852 } else {
853 assert(0);
854 return 0;
855 }
856 tssa->vars[ssa_var].no_val = no_val;
857 tssa->vars[ssa_var].alias = alias;
858 memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info));
859 return 1;
860 }
861 return 0;
862 }
863
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)864 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
865 {
866 zend_ssa_range tmp;
867 int def = tssa->vars[ssa_var].definition;
868
869 if (tssa->vars[ssa_var].alias == NO_ALIAS
870 && zend_inference_propagate_range(op_array, tssa, tssa_opcodes[def], &tssa->ops[def], ssa_var, &tmp)) {
871 tssa->var_info[ssa_var].range.min = tmp.min;
872 tssa->var_info[ssa_var].range.max = tmp.max;
873 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
874 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
875 tssa->var_info[ssa_var].has_range = 1;
876 }
877 }
878
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)879 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
880 {
881 int def;
882 zend_ssa_op *op;
883 zend_ssa_var_info *info;
884 unsigned int no_val;
885 zend_ssa_alias_kind alias;
886
887 def = tssa->vars[ssa_var].definition;
888 if (def >= 0) {
889 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
890 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
891 if (tssa->ops[def].op1_def == ssa_var) {
892 no_val = ssa->vars[op->op1_def].no_val;
893 alias = ssa->vars[op->op1_def].alias;
894 info = ssa->var_info + op->op1_def;
895 } else if (tssa->ops[def].op2_def == ssa_var) {
896 no_val = ssa->vars[op->op2_def].no_val;
897 alias = ssa->vars[op->op2_def].alias;
898 info = ssa->var_info + op->op2_def;
899 } else if (tssa->ops[def].result_def == ssa_var) {
900 no_val = ssa->vars[op->result_def].no_val;
901 alias = ssa->vars[op->result_def].alias;
902 info = ssa->var_info + op->result_def;
903 } else {
904 assert(0);
905 return;
906 }
907
908 tssa->vars[ssa_var].no_val = no_val;
909 tssa->vars[ssa_var].alias = alias;
910
911 if (!(info->type & MAY_BE_REF)) {
912 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
913 }
914
915 if (info->has_range) {
916 if (tssa->var_info[ssa_var].has_range) {
917 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
918 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
919 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
920 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
921 } else {
922 tssa->var_info[ssa_var].has_range = 1;
923 tssa->var_info[ssa_var].range = info->range;
924 }
925 }
926 }
927 }
928
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)929 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
930 {
931 int def;
932 zend_ssa_op *op;
933 zend_ssa_var_info *info;
934
935 def = tssa->vars[ssa_var].definition;
936 if (def >= 0) {
937 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
938 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
939 if (tssa->ops[def].op1_def == ssa_var) {
940 info = ssa->var_info + op->op1_def;
941 } else if (tssa->ops[def].op2_def == ssa_var) {
942 info = ssa->var_info + op->op2_def;
943 } else if (tssa->ops[def].result_def == ssa_var) {
944 info = ssa->var_info + op->result_def;
945 } else {
946 assert(0);
947 return 0;
948 }
949 tssa->var_info[ssa_var].type &= info->type;
950 if (info->ce) {
951 if (tssa->var_info[ssa_var].ce) {
952 if (tssa->var_info[ssa_var].ce != info->ce) {
953 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
954 /* everything fine */
955 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
956 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
957 } else {
958 // TODO: classes may implement the same interface ???
959 //ZEND_UNREACHABLE();
960 }
961 }
962 tssa->var_info[ssa_var].is_instanceof =
963 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
964 } else {
965 tssa->var_info[ssa_var].ce = info->ce;
966 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
967 }
968 }
969 if (info->has_range) {
970 if (tssa->var_info[ssa_var].has_range) {
971 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
972 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
973 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
974 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
975 } else {
976 tssa->var_info[ssa_var].has_range = 1;
977 tssa->var_info[ssa_var].range = info->range;
978 }
979 }
980 return 1;
981 }
982 return 0;
983 }
984
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)985 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
986 {
987 while (1) {
988 if (p->op == ZEND_JIT_TRACE_VM) {
989 if (p->opline->opcode == ZEND_DO_UCALL
990 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
991 || p->opline->opcode == ZEND_DO_FCALL) {
992 if (p->opline->result_type != IS_UNUSED) {
993 return ssa_op->result_def;
994 }
995 }
996 return -1;
997 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
998 /*skip */
999 } else {
1000 return -1;
1001 }
1002 p--;
1003 }
1004 }
1005
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1006 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1007 {
1008 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1009 p--;
1010 while (1) {
1011 if (p->op == ZEND_JIT_TRACE_VM) {
1012 if (p->opline->opcode == ZEND_INIT_FCALL
1013 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1014 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1015 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1016 || p->opline->opcode == ZEND_INIT_USER_CALL
1017 || p->opline->opcode == ZEND_NEW
1018 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1019 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
1020 return p->opline;
1021 }
1022 return NULL;
1023 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1024 /*skip */
1025 } else {
1026 return NULL;
1027 }
1028 p--;
1029 }
1030 } else {
1031 const zend_op *opline = NULL;
1032 int call_level = 0;
1033
1034 p++;
1035 while (1) {
1036 if (p->op == ZEND_JIT_TRACE_VM) {
1037 opline = p->opline;
1038 break;
1039 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1040 call_level++;
1041 /*skip */
1042 } else {
1043 return NULL;
1044 }
1045 p--;
1046 }
1047 if (opline) {
1048 while (opline > op_array->opcodes) {
1049 opline--;
1050 switch (opline->opcode) {
1051 case ZEND_INIT_FCALL:
1052 case ZEND_INIT_FCALL_BY_NAME:
1053 case ZEND_INIT_NS_FCALL_BY_NAME:
1054 case ZEND_INIT_METHOD_CALL:
1055 case ZEND_INIT_DYNAMIC_CALL:
1056 case ZEND_INIT_STATIC_METHOD_CALL:
1057 case ZEND_INIT_USER_CALL:
1058 case ZEND_NEW:
1059 if (call_level == 0) {
1060 return opline;
1061 }
1062 call_level--;
1063 break;
1064 case ZEND_DO_FCALL:
1065 case ZEND_DO_ICALL:
1066 case ZEND_DO_UCALL:
1067 case ZEND_DO_FCALL_BY_NAME:
1068 case ZEND_CALLABLE_CONVERT:
1069 call_level++;
1070 break;
1071 }
1072 }
1073 }
1074 }
1075 return NULL;
1076 }
1077
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1078 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1079 {
1080 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1081 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1082 int idx = tssa->vars[var].definition;
1083
1084 if (idx >= 0) {
1085 if (tssa->ops[idx].op1_def == var) {
1086 const zend_op *opline = ssa_opcodes[idx];
1087 if (opline->opcode == ZEND_PRE_DEC
1088 || opline->opcode == ZEND_PRE_INC
1089 || opline->opcode == ZEND_POST_DEC
1090 || opline->opcode == ZEND_POST_INC) {
1091 if (tssa->ops[idx].op1_use >= 0
1092 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1093 return 0;
1094 }
1095 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1096 return 0;
1097 }
1098 return 1;
1099 } else if (opline->opcode == ZEND_ASSIGN_OP
1100 && (opline->extended_value == ZEND_ADD
1101 || opline->extended_value == ZEND_SUB
1102 || opline->extended_value == ZEND_MUL)) {
1103 if ((opline->op2_type & (IS_VAR|IS_CV))
1104 && tssa->ops[idx].op2_use >= 0
1105 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1106 return 0;
1107 }
1108 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1109 return 0;
1110 }
1111 if (opline->op2_type == IS_CONST) {
1112 zval *zv = RT_CONSTANT(opline, opline->op2);
1113 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1114 return 0;
1115 }
1116 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1117 return 0;
1118 }
1119 return 1;
1120 }
1121 }
1122 if (tssa->ops[idx].result_def == var) {
1123 const zend_op *opline = ssa_opcodes[idx];
1124 if (opline->opcode == ZEND_ADD
1125 || opline->opcode == ZEND_SUB
1126 || opline->opcode == ZEND_MUL) {
1127 if ((opline->op1_type & (IS_VAR|IS_CV))
1128 && tssa->ops[idx].op1_use >= 0
1129 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1130 return 0;
1131 }
1132 if ((opline->op2_type & (IS_VAR|IS_CV))
1133 && tssa->ops[idx].op2_use >= 0
1134 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1135 return 0;
1136 }
1137 if (opline->op1_type == IS_CONST) {
1138 zval *zv = RT_CONSTANT(opline, opline->op1);
1139 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1140 return 0;
1141 }
1142 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1143 return 0;
1144 }
1145 if (opline->op2_type == IS_CONST) {
1146 zval *zv = RT_CONSTANT(opline, opline->op2);
1147 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1148 return 0;
1149 }
1150 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1151 return 0;
1152 }
1153 return 1;
1154 } else if (opline->opcode == ZEND_PRE_DEC
1155 || opline->opcode == ZEND_PRE_INC
1156 || opline->opcode == ZEND_POST_DEC
1157 || opline->opcode == ZEND_POST_INC) {
1158 if ((opline->op1_type & (IS_VAR|IS_CV))
1159 && tssa->ops[idx].op1_use >= 0
1160 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1161 return 0;
1162 }
1163 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1164 return 0;
1165 }
1166 return 1;
1167 }
1168 }
1169 }
1170 }
1171 return 0;
1172 }
1173
1174 typedef struct _zend_tssa {
1175 zend_ssa ssa;
1176 const zend_op **tssa_opcodes;
1177 int used_stack;
1178 } zend_tssa;
1179
1180 static const zend_op _nop_opcode = {0};
1181
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1182 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1183 {
1184 zend_ssa *tssa;
1185 zend_ssa_op *ssa_ops, *op;
1186 zend_ssa_var *ssa_vars;
1187 zend_ssa_var_info *ssa_var_info;
1188 const zend_op_array *op_array;
1189 const zend_op *opline;
1190 const zend_op **ssa_opcodes;
1191 zend_jit_trace_rec *p;
1192 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1193 zend_jit_trace_stack *stack;
1194 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1195 uint32_t optimization_level = 0;
1196 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1197 size_t frame_size, stack_top, stack_size, stack_bottom;
1198 zend_jit_op_array_trace_extension *jit_extension;
1199 zend_ssa *ssa;
1200 zend_jit_trace_stack_frame *frame, *top, *call;
1201 zend_ssa_var_info return_value_info;
1202
1203 /* 1. Count number of TSSA opcodes;
1204 * Count number of activation frames;
1205 * Calculate size of abstract stack;
1206 * Construct regular SSA for involved op_array */
1207 op_array = trace_buffer->op_array;
1208 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1209 stack_bottom = 0;
1210 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1211 ssa_ops_count = 0;
1212 call_level = 0;
1213 level = 0;
1214 num_op_arrays = 0;
1215 /* Remember op_array to cleanup */
1216 op_arrays[num_op_arrays++] = op_array;
1217 /* Build SSA */
1218 ssa = zend_jit_trace_build_ssa(op_array, script);
1219 for (;;p++) {
1220 if (p->op == ZEND_JIT_TRACE_VM) {
1221 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1222 const zend_op *opline = p->opline;
1223
1224 switch (opline->opcode) {
1225 case ZEND_INCLUDE_OR_EVAL:
1226 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1227 break;
1228 case ZEND_FETCH_R:
1229 case ZEND_FETCH_W:
1230 case ZEND_FETCH_RW:
1231 case ZEND_FETCH_FUNC_ARG:
1232 case ZEND_FETCH_IS:
1233 case ZEND_FETCH_UNSET:
1234 case ZEND_UNSET_VAR:
1235 case ZEND_ISSET_ISEMPTY_VAR:
1236 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1237 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1238 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1239 !op_array->function_name) {
1240 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1241 }
1242 break;
1243 }
1244 }
1245 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1246 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1247 call_level++;
1248 stack_top += zend_jit_trace_frame_size(p->op_array);
1249 if (stack_top > stack_size) {
1250 stack_size = stack_top;
1251 }
1252 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1253 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1254 if (p->func
1255 && p->func != (zend_function*)&zend_pass_function
1256 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1257 || zend_string_equals_literal(p->func->common.function_name, "compact")
1258 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1259 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1260 }
1261 }
1262 frame_size = zend_jit_trace_frame_size(p->op_array);
1263 if (call_level == 0) {
1264 if (stack_top + frame_size > stack_size) {
1265 stack_size = stack_top + frame_size;
1266 }
1267 } else {
1268 call_level--;
1269 stack_top -= frame_size;
1270 }
1271 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1272 op_array = p->op_array;
1273 if (call_level == 0) {
1274 stack_top += zend_jit_trace_frame_size(op_array);
1275 if (stack_top > stack_size) {
1276 stack_size = stack_top;
1277 }
1278 } else {
1279 call_level--;
1280 }
1281 level++;
1282 jit_extension =
1283 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1284 ssa = &jit_extension->func_info.ssa;
1285 if (ssa->cfg.blocks_count) {
1286 /* pass */
1287 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1288 /* Too many functions in single trace */
1289 *num_op_arrays_ptr = num_op_arrays;
1290 return NULL;
1291 } else {
1292 /* Remember op_array to cleanup */
1293 op_arrays[num_op_arrays++] = op_array;
1294 /* Build SSA */
1295 ssa = zend_jit_trace_build_ssa(op_array, script);
1296 }
1297 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1298 if (level == 0) {
1299 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1300 jit_extension =
1301 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1302 ssa = &jit_extension->func_info.ssa;
1303 if (ssa->cfg.blocks_count) {
1304 /* pass */
1305 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1306 /* Too many functions in single trace */
1307 *num_op_arrays_ptr = num_op_arrays;
1308 return NULL;
1309 } else {
1310 /* Remember op_array to cleanup */
1311 op_arrays[num_op_arrays++] = op_array;
1312 /* Build SSA */
1313 ssa = zend_jit_trace_build_ssa(op_array, script);
1314 }
1315 } else {
1316 stack_top -= zend_jit_trace_frame_size(op_array);
1317 level--;
1318 }
1319 op_array = p->op_array;
1320 } else if (p->op == ZEND_JIT_TRACE_END) {
1321 break;
1322 }
1323 }
1324 *num_op_arrays_ptr = num_op_arrays;
1325
1326 /* Allocate space for abstract stack */
1327 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1328
1329 /* 2. Construct TSSA */
1330 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1331 tssa->cfg.flags = ZEND_SSA_TSSA;
1332 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1333 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1334 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1335
1336 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1337 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1338 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1339 tssa->cfg.blocks_count = 2;
1340 tssa->cfg.edges_count = 2;
1341
1342 tssa->cfg.predecessors[0] = 0;
1343 tssa->cfg.predecessors[1] = 1;
1344
1345 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1346 tssa->cfg.blocks[0].successors_count = 1;
1347 tssa->cfg.blocks[0].predecessors_count = 0;
1348 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1349 tssa->cfg.blocks[0].successors[0] = 1;
1350
1351 tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1352 tssa->cfg.blocks[1].successors_count = 1;
1353 tssa->cfg.blocks[1].predecessors_count = 2;
1354 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1355 tssa->cfg.blocks[1].successors[1] = 1;
1356 } else {
1357 tssa->cfg.blocks_count = 1;
1358 tssa->cfg.edges_count = 0;
1359
1360 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1361 tssa->cfg.blocks[0].successors_count = 0;
1362 tssa->cfg.blocks[0].predecessors_count = 0;
1363 }
1364 ((zend_tssa*)tssa)->used_stack = -1;
1365
1366 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1367 return tssa;
1368 }
1369
1370 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1371 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1372 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1373 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1374 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1375
1376 op_array = trace_buffer->op_array;
1377 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1378 ssa_vars_count = op_array->last_var;
1379 } else {
1380 ssa_vars_count = op_array->last_var + op_array->T;
1381 }
1382 stack = frame->stack;
1383 for (i = 0; i < ssa_vars_count; i++) {
1384 SET_STACK_VAR(stack, i, i);
1385 }
1386
1387 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1388 // TODO: For tracing, it's possible, to create pseudo Phi functions
1389 // at the end of loop, without this additional pass (like LuaJIT) ???
1390 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1391 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1392 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1393 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1394 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1395 }
1396
1397 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1398 idx = 0;
1399 level = 0;
1400 for (;;p++) {
1401 if (p->op == ZEND_JIT_TRACE_VM) {
1402 opline = p->opline;
1403 ssa_opcodes[idx] = opline;
1404 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1405 idx++;
1406 len = zend_jit_trace_op_len(p->opline);
1407 while (len > 1) {
1408 opline++;
1409 ssa_opcodes[idx] = opline;
1410 if (opline->opcode != ZEND_OP_DATA) {
1411 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1412 }
1413 idx++;
1414 len--;
1415 }
1416 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1417 frame = zend_jit_trace_call_frame(frame, op_array);
1418 stack = frame->stack;
1419 op_array = p->op_array;
1420 level++;
1421 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1422 return NULL;
1423 }
1424 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1425 for (i = 0; i < op_array->last_var; i++) {
1426 SET_STACK_VAR(stack, i, ssa_vars_count++);
1427 }
1428 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1429 op_array = p->op_array;
1430 frame = zend_jit_trace_ret_frame(frame, op_array);
1431 stack = frame->stack;
1432 if (level == 0) {
1433 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1434 return NULL;
1435 }
1436 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1437 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1438 SET_STACK_VAR(stack, i, ssa_vars_count++);
1439 }
1440 } else {
1441 level--;
1442 }
1443 } else if (p->op == ZEND_JIT_TRACE_END) {
1444 break;
1445 }
1446 }
1447
1448 op_array = trace_buffer->op_array;
1449 tssa->vars_count = ssa_vars_count;
1450 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1451 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1452 vars_count = op_array->last_var;
1453 } else {
1454 vars_count = op_array->last_var + op_array->T;
1455 }
1456 i = 0;
1457 while (i < vars_count) {
1458 ssa_vars[i].var = i;
1459 ssa_vars[i].scc = -1;
1460 ssa_vars[i].definition = -1;
1461 ssa_vars[i].use_chain = -1;
1462 i++;
1463 }
1464 while (i < tssa->vars_count) {
1465 ssa_vars[i].var = -1;
1466 ssa_vars[i].scc = -1;
1467 ssa_vars[i].definition = -1;
1468 ssa_vars[i].use_chain = -1;
1469 i++;
1470 }
1471
1472 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1473 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1474 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1475 /* Update Phi sources */
1476 zend_ssa_phi *phi = tssa->blocks[1].phis;
1477
1478 while (phi) {
1479 phi->sources[1] = STACK_VAR(stack, phi->var);
1480 ssa_vars[phi->ssa_var].var = phi->var;
1481 ssa_vars[phi->ssa_var].definition_phi = phi;
1482 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1483 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1484 phi = phi->next;
1485 }
1486 }
1487
1488 /* 3. Compute use-def chains */
1489 idx = (ssa_ops_count - 1);
1490 op = ssa_ops + idx;
1491 while (idx >= 0) {
1492 opline = ssa_opcodes[idx];
1493 if (op->op1_use >= 0) {
1494 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1495 ssa_vars[op->op1_use].use_chain = idx;
1496 }
1497 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1498 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1499 ssa_vars[op->op2_use].use_chain = idx;
1500 }
1501 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1502 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1503 ssa_vars[op->result_use].use_chain = idx;
1504 }
1505 if (op->op1_def >= 0) {
1506 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1507 ssa_vars[op->op1_def].definition = idx;
1508 }
1509 if (op->op2_def >= 0) {
1510 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1511 ssa_vars[op->op2_def].definition = idx;
1512 }
1513 if (op->result_def >= 0) {
1514 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1515 ssa_vars[op->result_def].definition = idx;
1516 }
1517 op--;
1518 idx--;
1519 }
1520
1521 /* 4. Type inference */
1522 op_array = trace_buffer->op_array;
1523 jit_extension =
1524 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1525 ssa = &jit_extension->func_info.ssa;
1526
1527 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1528
1529 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1530 i = 0;
1531 while (i < op_array->last_var) {
1532 if (i < op_array->num_args) {
1533 if (ssa->var_info
1534 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1535 /* pass */
1536 } else {
1537 if (ssa->vars) {
1538 ssa_vars[i].no_val = ssa->vars[i].no_val;
1539 ssa_vars[i].alias = ssa->vars[i].alias;
1540 } else {
1541 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1542 }
1543 if (op_array->arg_info && i < trace_buffer[1].opline - op_array->opcodes) {
1544 zend_arg_info *arg_info = &op_array->arg_info[i];
1545 zend_class_entry *ce;
1546 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1547
1548 if (ZEND_ARG_SEND_MODE(arg_info)) {
1549 tmp |= MAY_BE_REF;
1550 }
1551 ssa_var_info[i].type = tmp;
1552 ssa_var_info[i].ce = ce;
1553 ssa_var_info[i].is_instanceof = 1;
1554 } else {
1555 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1556 }
1557 }
1558 } else {
1559 if (ssa->vars) {
1560 ssa_vars[i].no_val = ssa->vars[i].no_val;
1561 ssa_vars[i].alias = ssa->vars[i].alias;
1562 } else {
1563 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1564 }
1565 if (ssa_vars[i].alias == NO_ALIAS) {
1566 ssa_var_info[i].type = MAY_BE_UNDEF;
1567 } else {
1568 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1569 }
1570 }
1571 i++;
1572 }
1573 } else {
1574 int parent_vars_count = 0;
1575 zend_jit_trace_stack *parent_stack = NULL;
1576
1577 i = 0;
1578 if (parent_trace) {
1579 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1580 op_array->last_var + op_array->T);
1581 if (parent_vars_count) {
1582 parent_stack =
1583 zend_jit_traces[parent_trace].stack_map +
1584 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1585 }
1586 }
1587 while (i < op_array->last_var + op_array->T) {
1588 if (!ssa->var_info
1589 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1590 if (ssa->vars && i < ssa->vars_count) {
1591 ssa_vars[i].alias = ssa->vars[i].alias;
1592 } else {
1593 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1594 }
1595 if (i < op_array->last_var) {
1596 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1597 } else {
1598 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1599 }
1600 }
1601 if (i < parent_vars_count) {
1602 /* Initialize TSSA variable from parent trace */
1603 uint8_t op_type = STACK_TYPE(parent_stack, i);
1604
1605 if (op_type != IS_UNKNOWN) {
1606 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1607 if (!ssa_var_info[i].type
1608 && op_type == IS_UNDEF
1609 && i >= op_array->last_var) {
1610 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1611 ssa_var_info[i].type |= MAY_BE_UNDEF;
1612 }
1613 }
1614 }
1615 i++;
1616 }
1617 }
1618
1619 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1620 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1621 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1622 /* Propagate initial value through Phi functions */
1623 zend_ssa_phi *phi = tssa->blocks[1].phis;
1624
1625 while (phi) {
1626 if (!ssa->var_info
1627 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) {
1628 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1629 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1630 }
1631 phi = phi->next;
1632 }
1633 }
1634
1635 frame = JIT_G(current_frame);
1636 top = zend_jit_trace_call_frame(frame, op_array);
1637 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1638 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1639 frame->used_stack = 0;
1640 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1641 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
1642 }
1643 memset(&return_value_info, 0, sizeof(return_value_info));
1644
1645 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1646 max_used_stack = used_stack = 0;
1647 } else {
1648 max_used_stack = used_stack = -1;
1649 }
1650
1651 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1652 idx = 0;
1653 level = 0;
1654 opline = NULL;
1655 for (;;p++) {
1656 if (p->op == ZEND_JIT_TRACE_VM) {
1657 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1658 uint8_t val_type = IS_UNKNOWN;
1659 // zend_class_entry *op1_ce = NULL;
1660 zend_class_entry *op2_ce = NULL;
1661
1662 opline = p->opline;
1663
1664 op1_type = orig_op1_type = p->op1_type;
1665 op2_type = orig_op2_type = p->op2_type;
1666 op3_type = p->op3_type;
1667 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1668 op1_type = IS_UNKNOWN;
1669 }
1670 if (op1_type != IS_UNKNOWN) {
1671 op1_type &= ~IS_TRACE_PACKED;
1672 }
1673 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1674 op2_type = IS_UNKNOWN;
1675 }
1676 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1677 op3_type = IS_UNKNOWN;
1678 }
1679
1680 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1681 // op1_ce = (zend_class_entry*)(p+1)->ce;
1682 p++;
1683 }
1684 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1685 op2_ce = (zend_class_entry*)(p+1)->ce;
1686 p++;
1687 }
1688 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1689 val_type = (p+1)->op1_type;
1690 p++;
1691 }
1692
1693 switch (opline->opcode) {
1694 case ZEND_ASSIGN_OP:
1695 if (opline->extended_value == ZEND_POW
1696 || opline->extended_value == ZEND_DIV) {
1697 // TODO: check for division by zero ???
1698 break;
1699 }
1700 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1701 break;
1702 }
1703 ADD_OP1_TRACE_GUARD();
1704 ADD_OP2_TRACE_GUARD();
1705 break;
1706 case ZEND_ASSIGN_DIM_OP:
1707 if (opline->extended_value == ZEND_POW
1708 || opline->extended_value == ZEND_DIV) {
1709 // TODO: check for division by zero ???
1710 break;
1711 }
1712 if (opline->result_type != IS_UNUSED) {
1713 break;
1714 }
1715 if (op3_type != IS_UNKNOWN
1716 && !zend_jit_supported_binary_op(
1717 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1718 break;
1719 }
1720 ZEND_FALLTHROUGH;
1721 case ZEND_ASSIGN_DIM:
1722 if (opline->op1_type == IS_CV) {
1723 if ((opline+1)->op1_type == IS_CV
1724 && (opline+1)->op1.var == opline->op1.var) {
1725 /* skip $a[x] = $a; */
1726 break;
1727 }
1728 ADD_OP1_DATA_TRACE_GUARD();
1729 ADD_OP2_TRACE_GUARD();
1730 ADD_OP1_TRACE_GUARD();
1731 } else if (orig_op1_type != IS_UNKNOWN
1732 && (orig_op1_type & IS_TRACE_INDIRECT)
1733 && opline->result_type == IS_UNUSED) {
1734 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1735 ADD_OP1_DATA_TRACE_GUARD();
1736 }
1737 ADD_OP2_TRACE_GUARD();
1738 }
1739 if (op1_type == IS_ARRAY
1740 && ((opline->op2_type == IS_CONST
1741 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1742 || (opline->op2_type != IS_CONST
1743 && op2_type == IS_LONG))) {
1744
1745 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1746 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1747
1748 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1749 info->type |= MAY_BE_PACKED_GUARD;
1750 info->type &= ~MAY_BE_ARRAY_PACKED;
1751 }
1752 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1753 && val_type != IS_UNKNOWN
1754 && val_type != IS_UNDEF) {
1755 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1756
1757 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1758 info->type |= MAY_BE_PACKED_GUARD;
1759 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1760 }
1761 }
1762 }
1763 break;
1764 case ZEND_ASSIGN_OBJ_OP:
1765 if (opline->extended_value == ZEND_POW
1766 || opline->extended_value == ZEND_DIV) {
1767 // TODO: check for division by zero ???
1768 break;
1769 }
1770 if (opline->result_type != IS_UNUSED) {
1771 break;
1772 }
1773 ZEND_FALLTHROUGH;
1774 case ZEND_ASSIGN_OBJ:
1775 case ZEND_PRE_INC_OBJ:
1776 case ZEND_PRE_DEC_OBJ:
1777 case ZEND_POST_INC_OBJ:
1778 case ZEND_POST_DEC_OBJ:
1779 if (opline->op2_type != IS_CONST
1780 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1781 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1782 break;
1783 }
1784 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1785 if (opline->op1_type == IS_CV
1786 && (opline+1)->op1_type == IS_CV
1787 && (opline+1)->op1.var == opline->op1.var) {
1788 /* skip $a->prop += $a; */
1789 break;
1790 }
1791 ADD_OP1_DATA_TRACE_GUARD();
1792 }
1793 ADD_OP1_TRACE_GUARD();
1794 break;
1795 case ZEND_CONCAT:
1796 case ZEND_FAST_CONCAT:
1797 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1798 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1799 ADD_OP2_TRACE_GUARD();
1800 ADD_OP1_TRACE_GUARD();
1801 }
1802 break;
1803 case ZEND_IS_EQUAL:
1804 case ZEND_IS_NOT_EQUAL:
1805 case ZEND_IS_SMALLER:
1806 case ZEND_IS_SMALLER_OR_EQUAL:
1807 case ZEND_CASE:
1808 case ZEND_IS_IDENTICAL:
1809 case ZEND_IS_NOT_IDENTICAL:
1810 case ZEND_CASE_STRICT:
1811 case ZEND_BW_OR:
1812 case ZEND_BW_AND:
1813 case ZEND_BW_XOR:
1814 case ZEND_SL:
1815 case ZEND_SR:
1816 case ZEND_MOD:
1817 case ZEND_ADD:
1818 case ZEND_SUB:
1819 case ZEND_MUL:
1820 // case ZEND_DIV: // TODO: check for division by zero ???
1821 ADD_OP2_TRACE_GUARD();
1822 ZEND_FALLTHROUGH;
1823 case ZEND_ECHO:
1824 case ZEND_STRLEN:
1825 case ZEND_COUNT:
1826 case ZEND_QM_ASSIGN:
1827 case ZEND_FE_RESET_R:
1828 ADD_OP1_TRACE_GUARD();
1829 break;
1830 case ZEND_FE_FETCH_R:
1831 ADD_OP1_TRACE_GUARD();
1832 if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) {
1833
1834 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1835
1836 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1837 info->type |= MAY_BE_PACKED_GUARD;
1838 if (orig_op1_type & IS_TRACE_PACKED) {
1839 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1840 } else {
1841 info->type &= ~MAY_BE_ARRAY_PACKED;
1842 }
1843 }
1844 }
1845 break;
1846 case ZEND_VERIFY_RETURN_TYPE:
1847 if (opline->op1_type == IS_UNUSED) {
1848 /* Always throws */
1849 break;
1850 }
1851 if (opline->op1_type == IS_CONST) {
1852 /* TODO Different instruction format, has return value */
1853 break;
1854 }
1855 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1856 /* Not worth bothering with */
1857 break;
1858 }
1859 ADD_OP1_TRACE_GUARD();
1860 break;
1861 case ZEND_FETCH_DIM_FUNC_ARG:
1862 if (!frame
1863 || !frame->call
1864 || !frame->call->func
1865 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1866 break;
1867 }
1868 ADD_OP2_TRACE_GUARD();
1869 ADD_OP1_TRACE_GUARD();
1870 break;
1871 case ZEND_PRE_INC:
1872 case ZEND_PRE_DEC:
1873 case ZEND_POST_INC:
1874 case ZEND_POST_DEC:
1875 if (opline->op1_type != IS_CV) {
1876 break;
1877 }
1878 ADD_OP1_TRACE_GUARD();
1879 break;
1880 case ZEND_ASSIGN:
1881 if (opline->op1_type != IS_CV) {
1882 break;
1883 }
1884 ADD_OP2_TRACE_GUARD();
1885 if (op1_type != IS_UNKNOWN
1886 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1887 ADD_OP1_TRACE_GUARD();
1888 }
1889 break;
1890 case ZEND_CAST:
1891 if (opline->extended_value != op1_type) {
1892 break;
1893 }
1894 ADD_OP1_TRACE_GUARD();
1895 break;
1896 case ZEND_JMPZ:
1897 case ZEND_JMPNZ:
1898 case ZEND_JMPZ_EX:
1899 case ZEND_JMPNZ_EX:
1900 case ZEND_BOOL:
1901 case ZEND_BOOL_NOT:
1902 ADD_OP1_TRACE_GUARD();
1903 break;
1904 case ZEND_ISSET_ISEMPTY_CV:
1905 if ((opline->extended_value & ZEND_ISEMPTY)) {
1906 // TODO: support for empty() ???
1907 break;
1908 }
1909 ADD_OP1_TRACE_GUARD();
1910 break;
1911 case ZEND_IN_ARRAY:
1912 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
1913 break;
1914 }
1915 ADD_OP1_TRACE_GUARD();
1916 break;
1917 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
1918 if ((opline->extended_value & ZEND_ISEMPTY)) {
1919 // TODO: support for empty() ???
1920 break;
1921 }
1922 ZEND_FALLTHROUGH;
1923 case ZEND_FETCH_DIM_R:
1924 case ZEND_FETCH_DIM_IS:
1925 case ZEND_FETCH_LIST_R:
1926 ADD_OP1_TRACE_GUARD();
1927 ADD_OP2_TRACE_GUARD();
1928
1929 if (op1_type == IS_ARRAY
1930 && opline->op1_type != IS_CONST
1931 && ((opline->op2_type == IS_CONST
1932 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1933 || (opline->op2_type != IS_CONST
1934 && op2_type == IS_LONG))) {
1935
1936 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1937
1938 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1939 info->type |= MAY_BE_PACKED_GUARD;
1940 if (orig_op1_type & IS_TRACE_PACKED) {
1941 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1942 } else {
1943 info->type &= ~MAY_BE_ARRAY_PACKED;
1944 }
1945 }
1946 }
1947 break;
1948 case ZEND_FETCH_DIM_W:
1949 case ZEND_FETCH_DIM_RW:
1950 // case ZEND_FETCH_DIM_UNSET:
1951 case ZEND_FETCH_LIST_W:
1952 if (opline->op1_type != IS_CV
1953 && (orig_op1_type == IS_UNKNOWN
1954 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
1955 break;
1956 }
1957 ADD_OP1_TRACE_GUARD();
1958 ADD_OP2_TRACE_GUARD();
1959 if (op1_type == IS_ARRAY
1960 && !(orig_op1_type & IS_TRACE_PACKED)
1961 && ((opline->op2_type == IS_CONST
1962 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1963 || (opline->op2_type != IS_CONST
1964 && op2_type == IS_LONG))) {
1965
1966 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1967
1968 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1969 info->type |= MAY_BE_PACKED_GUARD;
1970 info->type &= ~MAY_BE_ARRAY_PACKED;
1971 }
1972 }
1973 break;
1974 case ZEND_SEND_VAL_EX:
1975 case ZEND_SEND_VAR_EX:
1976 case ZEND_SEND_VAR_NO_REF_EX:
1977 if (opline->op2_type == IS_CONST) {
1978 /* Named parameters not supported in JIT */
1979 break;
1980 }
1981 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
1982 goto propagate_arg;
1983 }
1984 ZEND_FALLTHROUGH;
1985 case ZEND_SEND_VAL:
1986 case ZEND_SEND_VAR:
1987 case ZEND_SEND_VAR_NO_REF:
1988 case ZEND_SEND_FUNC_ARG:
1989 if (opline->op2_type == IS_CONST) {
1990 /* Named parameters not supported in JIT */
1991 break;
1992 }
1993 ADD_OP1_TRACE_GUARD();
1994 propagate_arg:
1995 /* Propagate argument type */
1996 if (frame->call
1997 && frame->call->func
1998 && frame->call->func->type == ZEND_USER_FUNCTION
1999 && opline->op2.num <= frame->call->func->op_array.num_args) {
2000 uint32_t info;
2001
2002 if (opline->op1_type == IS_CONST) {
2003 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
2004 } else {
2005 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2006 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
2007 }
2008 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
2009 zend_arg_info *arg_info;
2010
2011 ZEND_ASSERT(frame->call->func->op_array.arg_info);
2012 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
2013 if (ZEND_TYPE_IS_SET(arg_info->type)) {
2014 zend_class_entry *ce;
2015 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2016 info &= tmp;
2017 if (!info) {
2018 break;
2019 }
2020 }
2021 }
2022 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2023 info |= MAY_BE_RCN;
2024 }
2025 if (info & MAY_BE_UNDEF) {
2026 info |= MAY_BE_NULL;
2027 info &= ~MAY_BE_UNDEF;
2028 }
2029 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2030 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2031 }
2032 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2033 }
2034 break;
2035 case ZEND_RETURN:
2036 ADD_OP1_TRACE_GUARD();
2037 /* Propagate return value types */
2038 if (opline->op1_type == IS_UNUSED) {
2039 return_value_info.type = MAY_BE_NULL;
2040 } else if (opline->op1_type == IS_CONST) {
2041 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2042 } else {
2043 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2044 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2045 if (return_value_info.type & MAY_BE_UNDEF) {
2046 return_value_info.type &= ~MAY_BE_UNDEF;
2047 return_value_info.type |= MAY_BE_NULL;
2048 }
2049 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2050 /* CVs are going to be destructed and the reference-counter
2051 of return value may be decremented to 1 */
2052 return_value_info.type |= MAY_BE_RC1;
2053 }
2054 return_value_info.type &= ~MAY_BE_GUARD;
2055 }
2056 break;
2057 case ZEND_CHECK_FUNC_ARG:
2058 if (!frame
2059 || !frame->call
2060 || !frame->call->func) {
2061 break;
2062 }
2063 if (opline->op2_type == IS_CONST
2064 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2065 /* Named parameters not supported in JIT */
2066 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2067 break;
2068 }
2069 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2070 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2071 } else {
2072 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2073 }
2074 break;
2075 case ZEND_FETCH_OBJ_FUNC_ARG:
2076 if (!frame
2077 || !frame->call
2078 || !frame->call->func
2079 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2080 break;
2081 }
2082 ZEND_FALLTHROUGH;
2083 case ZEND_FETCH_OBJ_R:
2084 case ZEND_FETCH_OBJ_IS:
2085 case ZEND_FETCH_OBJ_W:
2086 if (opline->op2_type != IS_CONST
2087 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2088 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2089 break;
2090 }
2091 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2092 ADD_OP1_TRACE_GUARD();
2093 }
2094 break;
2095 case ZEND_INIT_METHOD_CALL:
2096 if (opline->op2_type != IS_CONST
2097 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2098 break;
2099 }
2100 ADD_OP1_TRACE_GUARD();
2101 break;
2102 case ZEND_INIT_DYNAMIC_CALL:
2103 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2104 ADD_OP2_TRACE_GUARD();
2105 }
2106 break;
2107 case ZEND_SEND_ARRAY:
2108 case ZEND_SEND_UNPACK:
2109 case ZEND_CHECK_UNDEF_ARGS:
2110 case ZEND_INCLUDE_OR_EVAL:
2111 max_used_stack = used_stack = -1;
2112 break;
2113 case ZEND_TYPE_CHECK:
2114 if (opline->extended_value == MAY_BE_RESOURCE) {
2115 // TODO: support for is_resource() ???
2116 break;
2117 }
2118 if (op1_type != IS_UNKNOWN
2119 && (opline->extended_value == (1 << op1_type)
2120 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2121 /* add guards only for exact checks, to avoid code duplication */
2122 ADD_OP1_TRACE_GUARD();
2123 }
2124 break;
2125 case ZEND_ROPE_INIT:
2126 case ZEND_ROPE_ADD:
2127 case ZEND_ROPE_END:
2128 if (op2_type == IS_STRING) {
2129 ADD_OP2_TRACE_GUARD();
2130 }
2131 break;
2132 default:
2133 break;
2134 }
2135 len = zend_jit_trace_op_len(opline);
2136 if (ssa->var_info) {
2137 /* Add statically inferred ranges */
2138 if (ssa_ops[idx].op1_def >= 0) {
2139 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2140 }
2141 if (ssa_ops[idx].op2_def >= 0) {
2142 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2143 }
2144 if (ssa_ops[idx].result_def >= 0) {
2145 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2146 }
2147 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2148 if (ssa_ops[idx+1].op1_def >= 0) {
2149 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2150 }
2151 if (ssa_ops[idx+1].op2_def >= 0) {
2152 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2153 }
2154 if (ssa_ops[idx+1].result_def >= 0) {
2155 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2156 }
2157 }
2158 } else {
2159 if (ssa_ops[idx].op1_def >= 0) {
2160 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2161 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2162 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2163 }
2164 }
2165 if (ssa_ops[idx].op2_def >= 0) {
2166 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2167 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2168 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2169 }
2170 }
2171 if (ssa_ops[idx].result_def >= 0) {
2172 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2173 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2174 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2175 }
2176 }
2177 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2178 if (ssa_ops[idx+1].op1_def >= 0) {
2179 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2180 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2181 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2182 }
2183 }
2184 if (ssa_ops[idx+1].op2_def >= 0) {
2185 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2186 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2187 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2188 }
2189 }
2190 if (ssa_ops[idx+1].result_def >= 0) {
2191 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2192 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2193 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2194 }
2195 }
2196 }
2197 }
2198 if (opline->opcode == ZEND_RECV_INIT
2199 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2200 /* RECV_INIT always copy the constant */
2201 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2202 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2203 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2204 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2205 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2206 }
2207 } else {
2208 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2209 // TODO:
2210 assert(0);
2211 }
2212 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2213 && ssa_ops[idx].op1_def >= 0
2214 && op1_type == IS_ARRAY
2215 && (orig_op1_type & IS_TRACE_PACKED)
2216 && val_type != IS_UNKNOWN
2217 && val_type != IS_UNDEF
2218 && ((opline->op2_type == IS_CONST
2219 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2220 || (opline->op2_type != IS_CONST
2221 && op2_type == IS_LONG))) {
2222 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2223
2224 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2225 }
2226 }
2227 if (ssa->var_info) {
2228 /* Add statically inferred restrictions */
2229 if (ssa_ops[idx].op1_def >= 0) {
2230 if (opline->opcode == ZEND_SEND_VAR_EX
2231 && frame
2232 && frame->call
2233 && frame->call->func
2234 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2235 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2236 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2237 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2238 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2239 }
2240 } else {
2241 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2242 }
2243 }
2244 if (ssa_ops[idx].op2_def >= 0) {
2245 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2246 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2247 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2248 }
2249 }
2250 if (ssa_ops[idx].result_def >= 0) {
2251 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2252 }
2253 }
2254 idx++;
2255 while (len > 1) {
2256 opline++;
2257 if (opline->opcode != ZEND_OP_DATA) {
2258 if (ssa->var_info) {
2259 /* Add statically inferred ranges */
2260 if (ssa_ops[idx].op1_def >= 0) {
2261 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2262 }
2263 if (ssa_ops[idx].op2_def >= 0) {
2264 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2265 }
2266 if (ssa_ops[idx].result_def >= 0) {
2267 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2268 }
2269 } else {
2270 if (ssa_ops[idx].op1_def >= 0) {
2271 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2272 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2273 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2274 }
2275 }
2276 if (ssa_ops[idx].op2_def >= 0) {
2277 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2278 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2279 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2280 }
2281 }
2282 if (ssa_ops[idx].result_def >= 0) {
2283 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2284 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2285 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2286 }
2287 }
2288 }
2289 if (opline->opcode == ZEND_RECV_INIT
2290 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2291 /* RECV_INIT always copy the constant */
2292 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2293 } else {
2294 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2295 // TODO:
2296 assert(0);
2297 }
2298 }
2299 }
2300 if (ssa->var_info) {
2301 /* Add statically inferred restrictions */
2302 if (ssa_ops[idx].op1_def >= 0) {
2303 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2304 }
2305 if (ssa_ops[idx].op2_def >= 0) {
2306 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2307 }
2308 if (ssa_ops[idx].result_def >= 0) {
2309 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2310 }
2311 }
2312 idx++;
2313 len--;
2314 }
2315
2316 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2317 op_array = p->op_array;
2318 jit_extension =
2319 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2320 ssa = &jit_extension->func_info.ssa;
2321
2322 call = frame->call;
2323 if (!call) {
2324 /* Trace missed INIT_FCALL opcode */
2325 call = top;
2326 TRACE_FRAME_INIT(call, op_array, 0, 0);
2327 call->used_stack = 0;
2328 top = zend_jit_trace_call_frame(top, op_array);
2329 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2330 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
2331 }
2332 } else {
2333 ZEND_ASSERT(&call->func->op_array == op_array);
2334 }
2335 frame->call = call->prev;
2336 call->prev = frame;
2337 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2338 frame = call;
2339
2340 level++;
2341 i = 0;
2342 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2343 while (i < op_array->last_var) {
2344 ssa_vars[v].var = i;
2345 if (i < op_array->num_args) {
2346 if (ssa->var_info
2347 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2348 /* pass */
2349 } else {
2350 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2351 if (op_array->arg_info) {
2352 zend_arg_info *arg_info = &op_array->arg_info[i];
2353 zend_class_entry *ce;
2354 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2355
2356 if (ZEND_ARG_SEND_MODE(arg_info)) {
2357 tmp |= MAY_BE_REF;
2358 }
2359 ssa_var_info[v].type = tmp;
2360 ssa_var_info[v].ce = ce;
2361 ssa_var_info[v].is_instanceof = 1;
2362 } else {
2363 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2364 }
2365 }
2366 } else {
2367 if (ssa->vars) {
2368 ssa_vars[v].no_val = ssa->vars[i].no_val;
2369 ssa_vars[v].alias = ssa->vars[i].alias;
2370 } else {
2371 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2372 }
2373 if (ssa_vars[v].alias == NO_ALIAS) {
2374 ssa_var_info[v].type = MAY_BE_UNDEF;
2375 } else {
2376 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2377 }
2378 }
2379 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2380 && i < op_array->num_args) {
2381 /* Propagate argument type */
2382 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2383 }
2384 i++;
2385 v++;
2386 }
2387 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2388 op_array = p->op_array;
2389 jit_extension =
2390 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2391 ssa = &jit_extension->func_info.ssa;
2392 if (level == 0) {
2393 i = 0;
2394 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2395 while (i < op_array->last_var) {
2396 ssa_vars[v].var = i;
2397 if (!ssa->var_info
2398 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2399 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2400 }
2401 i++;
2402 v++;
2403 }
2404 while (i < op_array->last_var + op_array->T) {
2405 ssa_vars[v].var = i;
2406 if (!ssa->var_info
2407 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2408 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2409 }
2410 i++;
2411 v++;
2412 }
2413 if (return_value_info.type != 0) {
2414 zend_jit_trace_rec *q = p + 1;
2415 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2416 q++;
2417 }
2418 if (q->op == ZEND_JIT_TRACE_VM
2419 || (q->op == ZEND_JIT_TRACE_END
2420 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2421 const zend_op *opline = q->opline - 1;
2422 if (opline->result_type != IS_UNUSED) {
2423 ssa_var_info[
2424 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2425 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2426 }
2427 }
2428 memset(&return_value_info, 0, sizeof(return_value_info));
2429 }
2430 } else {
2431 level--;
2432 if (return_value_info.type != 0) {
2433 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2434 const zend_op *opline = (p+1)->opline - 1;
2435 if (opline->result_type != IS_UNUSED) {
2436 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2437 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2438 }
2439 }
2440 }
2441 memset(&return_value_info, 0, sizeof(return_value_info));
2442 }
2443 }
2444
2445 top = frame;
2446 if (frame->prev) {
2447 if (used_stack > 0) {
2448 used_stack -= frame->used_stack;
2449 }
2450 frame = frame->prev;
2451 ZEND_ASSERT(&frame->func->op_array == op_array);
2452 } else {
2453 max_used_stack = used_stack = -1;
2454 frame = zend_jit_trace_ret_frame(frame, op_array);
2455 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2456 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2457 frame->used_stack = 0;
2458 for (i = 0; i < op_array->last_var + op_array->T; i++) {
2459 SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1);
2460 }
2461 }
2462
2463 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2464 call = top;
2465 TRACE_FRAME_INIT(call, p->func, 0, 0);
2466 call->prev = frame->call;
2467 call->used_stack = 0;
2468 frame->call = call;
2469 top = zend_jit_trace_call_frame(top, p->op_array);
2470 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2471 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2472 SET_STACK_INFO(call->stack, i, -1);
2473 }
2474 }
2475 if (used_stack >= 0
2476 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2477 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2478 max_used_stack = used_stack = -1;
2479 } else {
2480 const zend_op *opline = (p-1)->opline;
2481
2482 switch (opline->opcode) {
2483 case ZEND_INIT_FCALL:
2484 case ZEND_INIT_FCALL_BY_NAME:
2485 case ZEND_INIT_NS_FCALL_BY_NAME:
2486 case ZEND_INIT_METHOD_CALL:
2487 case ZEND_INIT_DYNAMIC_CALL:
2488 //case ZEND_INIT_STATIC_METHOD_CALL:
2489 //case ZEND_INIT_USER_CALL:
2490 //case ZEND_NEW:
2491 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2492 used_stack += frame->used_stack;
2493 if (used_stack > max_used_stack) {
2494 max_used_stack = used_stack;
2495 }
2496 break;
2497 default:
2498 max_used_stack = used_stack = -1;
2499 }
2500 }
2501 }
2502 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2503 call = frame->call;
2504 if (call) {
2505 top = call;
2506 frame->call = call->prev;
2507 }
2508
2509 if (idx > 0
2510 && ssa_ops[idx-1].result_def >= 0
2511 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2512 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2513 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2514 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2515 opline->opcode == ZEND_DO_FCALL ||
2516 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2517
2518 if (opline->result_type != IS_UNDEF) {
2519 zend_class_entry *ce;
2520 const zend_function *func = p->func;
2521 zend_arg_info *ret_info = func->common.arg_info - 1;
2522 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2523
2524 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2525 }
2526 }
2527 } else if (p->op == ZEND_JIT_TRACE_END) {
2528 break;
2529 }
2530 }
2531
2532 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2533
2534 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2535 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2536 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2537 /* Propagate guards through Phi sources */
2538 zend_ssa_phi *phi = tssa->blocks[1].phis;
2539
2540 op_array = trace_buffer->op_array;
2541 jit_extension =
2542 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2543 ssa = &jit_extension->func_info.ssa;
2544
2545 while (phi) {
2546 uint32_t t = ssa_var_info[phi->ssa_var].type;
2547
2548 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2549 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2550 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2551
2552 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2553 if (!((t0 | t1) & MAY_BE_GUARD)) {
2554 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2555 }
2556 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2557 if (!(t1 & MAY_BE_GUARD)
2558 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2559 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2560 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2561 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2562 MAY_BE_GUARD;
2563 if (!(t0 & MAY_BE_ARRAY)) {
2564 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2565 }
2566 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2567 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2568 }
2569 ssa_var_info[phi->sources[0]].type = t0;
2570 ssa_var_info[phi->sources[0]].type = t0;
2571 }
2572 } else {
2573 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2574 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2575 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2576 MAY_BE_GUARD;
2577 if (!(t0 & MAY_BE_ARRAY)) {
2578 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2579 }
2580 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2581 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2582 }
2583 ssa_var_info[phi->sources[0]].type = t0;
2584 }
2585 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2586 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2587 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2588 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2589 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2590 MAY_BE_GUARD;
2591 if (!(t1 & MAY_BE_ARRAY)) {
2592 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2593 }
2594 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2595 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2596 }
2597 ssa_var_info[phi->sources[1]].type = t1;
2598 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2599 }
2600 }
2601 }
2602 t = ssa_var_info[phi->ssa_var].type;
2603 }
2604
2605 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2606 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2607 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2608
2609 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2610 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2611 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2612 }
2613 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2614 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2615 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2616 ssa_var_info[phi->sources[0]].type =
2617 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2618 }
2619 }
2620 }
2621 phi = phi->next;
2622 }
2623 }
2624
2625 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2626 if (parent_trace) {
2627 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2628 ZEND_JIT_TRACE_NUM,
2629 parent_trace,
2630 exit_num,
2631 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2632 trace_buffer->op_array->scope ? "::" : "",
2633 trace_buffer->op_array->function_name ?
2634 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2635 ZSTR_VAL(trace_buffer->op_array->filename),
2636 trace_buffer[1].opline->lineno);
2637 } else {
2638 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2639 ZEND_JIT_TRACE_NUM,
2640 zend_jit_trace_star_desc(trace_buffer->start),
2641 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2642 trace_buffer->op_array->scope ? "::" : "",
2643 trace_buffer->op_array->function_name ?
2644 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2645 ZSTR_VAL(trace_buffer->op_array->filename),
2646 trace_buffer[1].opline->lineno);
2647 }
2648 zend_jit_dump_trace(trace_buffer, tssa);
2649 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2650 uint32_t idx = trace_buffer[1].last;
2651 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2652 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2653 ZEND_JIT_TRACE_NUM,
2654 link_to);
2655 } else {
2656 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2657 ZEND_JIT_TRACE_NUM,
2658 zend_jit_trace_stop_description[trace_buffer->stop]);
2659 }
2660 }
2661
2662 return tssa;
2663 }
2664
zend_jit_close_var(zend_jit_trace_stack * stack,uint32_t n,int * start,int * end,uint8_t * flags,int line)2665 static void zend_jit_close_var(zend_jit_trace_stack *stack, uint32_t n, int *start, int *end, uint8_t *flags, int line)
2666 {
2667 int32_t var = STACK_VAR(stack, n);
2668
2669 if (var >= 0 && start[var] >= 0 && !(flags[var] & ZREG_LAST_USE)) {
2670 // TODO: shrink interval to last side exit ????
2671 end[var] = line;
2672 }
2673 }
2674
zend_jit_trace_use_var(int line,int var,int def,int use_chain,int * start,int * end,uint8_t * flags,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2675 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, int *start, int *end, uint8_t *flags, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2676 {
2677 ZEND_ASSERT(start[var] >= 0);
2678 ZEND_ASSERT(!(flags[var] & ZREG_LAST_USE));
2679 end[var] = line;
2680 if (def >= 0) {
2681 flags[var] |= ZREG_LAST_USE;
2682 } else if (use_chain < 0 && (flags[var] & (ZREG_LOAD|ZREG_STORE))) {
2683 flags[var] |= ZREG_LAST_USE;
2684 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2685 /* pass */
2686 } else if (op_array_ssa->vars) {
2687 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2688
2689 if (ssa->ops[line].op1_use == var) {
2690 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2691 flags[var] |= ZREG_LAST_USE;
2692 }
2693 } else if (ssa->ops[line].op2_use == var) {
2694 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2695 flags[var] |= ZREG_LAST_USE;
2696 }
2697 } else if (ssa->ops[line].result_use == var) {
2698 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2699 flags[var] |= ZREG_LAST_USE;
2700 }
2701 }
2702 }
2703 }
2704
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2705 static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2706 {
2707 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2708 zend_jit_trace_rec *p;
2709 const zend_op_array *op_array;
2710 zend_jit_op_array_trace_extension *jit_extension;
2711 const zend_ssa *op_array_ssa;
2712 const zend_ssa_op *ssa_op;
2713 int i, j, idx, count, level;
2714 int last_idx = -1;
2715 int *start, *end;
2716 uint8_t *flags;
2717 const zend_op_array **vars_op_array;
2718 zend_lifetime_interval **intervals, *list, *ival;
2719 void *checkpoint;
2720 zend_jit_trace_stack_frame *frame;
2721 zend_jit_trace_stack *stack;
2722 uint32_t parent_vars_count = parent_trace ?
2723 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2724 zend_jit_trace_stack *parent_stack = parent_trace ?
2725 zend_jit_traces[parent_trace].stack_map +
2726 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2727 ALLOCA_FLAG(use_heap);
2728
2729 ZEND_ASSERT(ssa->var_info != NULL);
2730
2731 start = do_alloca(sizeof(int) * ssa->vars_count * 2 +
2732 ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count),
2733 use_heap);
2734 if (!start) {
2735 return NULL;
2736 }
2737 end = start + ssa->vars_count;
2738 flags = (uint8_t*)(end + ssa->vars_count);
2739 checkpoint = zend_arena_checkpoint(CG(arena));
2740 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2741
2742 memset(start, -1, sizeof(int) * ssa->vars_count * 2);
2743 memset(flags, 0, sizeof(uint8_t) * ssa->vars_count);
2744 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2745
2746 op_array = trace_buffer->op_array;
2747 jit_extension =
2748 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2749 op_array_ssa = &jit_extension->func_info.ssa;
2750 frame = JIT_G(current_frame);
2751 frame->prev = NULL;
2752 frame->func = (const zend_function*)op_array;
2753 stack = frame->stack;
2754
2755 count = 0;
2756
2757 i = 0;
2758 j = op_array->last_var;
2759 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2760 j += op_array->T;
2761 }
2762 while (i < j) {
2763 SET_STACK_VAR(stack, i, i);
2764 vars_op_array[i] = op_array;
2765 /* We don't start intervals for variables used in Phi */
2766 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2767 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2768 && ssa->vars[i].alias == NO_ALIAS
2769 && zend_jit_var_supports_reg(ssa, i)) {
2770 start[i] = 0;
2771 if (i < parent_vars_count
2772 && STACK_REG(parent_stack, i) != ZREG_NONE
2773 && STACK_REG(parent_stack, i) < ZREG_NUM) {
2774 /* We will try to reuse register from parent trace */
2775 flags[i] = STACK_FLAGS(parent_stack, i);
2776 count += 2;
2777 } else {
2778 flags[i] = ZREG_LOAD;
2779 count++;
2780 }
2781 }
2782 i++;
2783 }
2784
2785 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2786 j = op_array->last_var + op_array->T;
2787 while (i < j) {
2788 SET_STACK_VAR(stack, i, -1);
2789 i++;
2790 }
2791 }
2792
2793 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2794 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2795 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2796 zend_ssa_phi *phi = ssa->blocks[1].phis;
2797
2798 while (phi) {
2799 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2800 vars_op_array[phi->ssa_var] = op_array;
2801 if (ssa->vars[phi->ssa_var].use_chain >= 0
2802 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2803 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2804 start[phi->ssa_var] = 0;
2805 count++;
2806 }
2807 phi = phi->next;
2808 }
2809 }
2810
2811 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2812 level = 0;
2813 ssa_op = ssa->ops;
2814 idx = 0;
2815 for (;;p++) {
2816 if (p->op == ZEND_JIT_TRACE_VM) {
2817 const zend_op *opline = p->opline;
2818 int len;
2819 bool support_opline;
2820
2821 support_opline =
2822 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2823
2824 if (support_opline
2825 && opline->opcode == ZEND_ASSIGN
2826 && opline->op1_type == IS_CV
2827 && ssa_op->op1_def >= 0
2828 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2829 /* avoid register allocation in case of possibility of indirect modification*/
2830 support_opline = 0;
2831 }
2832
2833 if (ssa_op->op1_use >= 0
2834 && start[ssa_op->op1_use] >= 0
2835 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2836 if (support_opline) {
2837 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2838 if (opline->op1_type != IS_CV) {
2839 if (opline->opcode == ZEND_CASE
2840 || opline->opcode == ZEND_CASE_STRICT
2841 || opline->opcode == ZEND_SWITCH_LONG
2842 || opline->opcode == ZEND_MATCH
2843 || opline->opcode == ZEND_FETCH_LIST_R
2844 || opline->opcode == ZEND_COPY_TMP
2845 || opline->opcode == ZEND_SWITCH_STRING
2846 || opline->opcode == ZEND_FE_FETCH_R
2847 || opline->opcode == ZEND_FE_FETCH_RW
2848 || opline->opcode == ZEND_FETCH_LIST_W
2849 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2850 || opline->opcode == ZEND_BIND_LEXICAL
2851 || opline->opcode == ZEND_ROPE_ADD) {
2852 /* The value is kept alive and may be used outside of the trace */
2853 flags[ssa_op->op1_use] |= ZREG_STORE;
2854 } else {
2855 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2856 }
2857 }
2858 } else {
2859 start[ssa_op->op1_use] = -1;
2860 end[ssa_op->op1_use] = -1;
2861 count--;
2862 }
2863 }
2864 if (ssa_op->op2_use >= 0
2865 && ssa_op->op2_use != ssa_op->op1_use
2866 && start[ssa_op->op2_use] >= 0
2867 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2868 if (support_opline) {
2869 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2870 if (opline->op2_type != IS_CV) {
2871 flags[ssa_op->op2_use] |= ZREG_LAST_USE;
2872 }
2873 } else {
2874 start[ssa_op->op2_use] = -1;
2875 end[ssa_op->op2_use] = -1;
2876 count--;
2877 }
2878 }
2879 if (ssa_op->result_use >= 0
2880 && ssa_op->result_use != ssa_op->op1_use
2881 && ssa_op->result_use != ssa_op->op2_use
2882 && start[ssa_op->result_use] >= 0
2883 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2884 if (support_opline) {
2885 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2886 } else {
2887 start[ssa_op->result_use] = -1;
2888 end[ssa_op->result_use] = -1;
2889 count--;
2890 }
2891 }
2892
2893 if (ssa_op->op1_def >= 0) {
2894 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2895 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2896 }
2897 if (ssa_op->op2_def >= 0) {
2898 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op2.var), start, end, flags, idx);
2899 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
2900 }
2901 if (ssa_op->result_def >= 0) {
2902 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
2903 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2904 }
2905
2906 if (support_opline) {
2907 if (ssa_op->result_def >= 0
2908 && (ssa->vars[ssa_op->result_def].use_chain >= 0
2909 || ssa->vars[ssa_op->result_def].phi_use_chain)
2910 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
2911 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
2912 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
2913 || opline->opcode == ZEND_PRE_INC
2914 || opline->opcode == ZEND_PRE_DEC
2915 || opline->opcode == ZEND_POST_INC
2916 || opline->opcode == ZEND_POST_DEC
2917 || opline->opcode == ZEND_ADD
2918 || opline->opcode == ZEND_SUB
2919 || opline->opcode == ZEND_MUL
2920 || opline->opcode == ZEND_FETCH_DIM_R
2921 || opline->opcode == ZEND_FETCH_CONSTANT) {
2922 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
2923 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
2924 start[ssa_op->result_def] = idx;
2925 vars_op_array[ssa_op->result_def] = op_array;
2926 count++;
2927 }
2928 }
2929 }
2930 if (ssa_op->op1_def >= 0
2931 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2932 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2933 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2934 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2935 start[ssa_op->op1_def] = idx;
2936 vars_op_array[ssa_op->op1_def] = op_array;
2937 count++;
2938 }
2939 if (ssa_op->op2_def >= 0
2940 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
2941 || ssa->vars[ssa_op->op2_def].phi_use_chain)
2942 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
2943 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)) {
2944 start[ssa_op->op2_def] = idx;
2945 vars_op_array[ssa_op->op2_def] = op_array;
2946 count++;
2947 }
2948 }
2949
2950 len = zend_jit_trace_op_len(opline);
2951 switch (opline->opcode) {
2952 case ZEND_ASSIGN_DIM:
2953 case ZEND_ASSIGN_OBJ:
2954 case ZEND_ASSIGN_STATIC_PROP:
2955 case ZEND_ASSIGN_DIM_OP:
2956 case ZEND_ASSIGN_OBJ_OP:
2957 case ZEND_ASSIGN_STATIC_PROP_OP:
2958 case ZEND_ASSIGN_OBJ_REF:
2959 case ZEND_ASSIGN_STATIC_PROP_REF:
2960 /* OP_DATA */
2961 ssa_op++;
2962 opline++;
2963 if (ssa_op->op1_use >= 0
2964 && start[ssa_op->op1_use] >= 0
2965 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2966 if (support_opline) {
2967 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa);
2968 if (opline->op1_type != IS_CV) {
2969 flags[ssa_op->op1_use] |= ZREG_LAST_USE;
2970 }
2971 } else {
2972 start[ssa_op->op1_use] = -1;
2973 end[ssa_op->op1_use] = -1;
2974 count--;
2975 }
2976 }
2977 if (ssa_op->op1_def >= 0) {
2978 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
2979 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2980 if (support_opline
2981 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2982 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2983 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2984 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
2985 start[ssa_op->op1_def] = idx;
2986 vars_op_array[ssa_op->op1_def] = op_array;
2987 count++;
2988 }
2989 }
2990 ssa_op++;
2991 opline++;
2992 idx+=2;
2993 break;
2994 case ZEND_RECV_INIT:
2995 ssa_op++;
2996 opline++;
2997 idx++;
2998 while (opline->opcode == ZEND_RECV_INIT) {
2999 /* RECV_INIT doesn't support registers */
3000 if (ssa_op->result_def >= 0) {
3001 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx);
3002 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3003 }
3004 ssa_op++;
3005 opline++;
3006 idx++;
3007 }
3008 break;
3009 case ZEND_BIND_GLOBAL:
3010 ssa_op++;
3011 opline++;
3012 idx++;
3013 while (opline->opcode == ZEND_BIND_GLOBAL) {
3014 /* BIND_GLOBAL doesn't support registers */
3015 if (ssa_op->op1_def >= 0) {
3016 zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx);
3017 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3018 }
3019 ssa_op++;
3020 opline++;
3021 idx++;
3022 }
3023 break;
3024 default:
3025 ssa_op += len;
3026 idx += len;
3027 break;
3028 }
3029 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3030 /* New call frames */
3031 zend_jit_trace_stack_frame *prev_frame = frame;
3032
3033 frame = zend_jit_trace_call_frame(frame, op_array);
3034 frame->prev = prev_frame;
3035 frame->func = (const zend_function*)p->op_array;
3036 stack = frame->stack;
3037 op_array = p->op_array;
3038 jit_extension =
3039 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3040 op_array_ssa = &jit_extension->func_info.ssa;
3041 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3042 for (i = 0; i < op_array->last_var; i++) {
3043 SET_STACK_VAR(stack, i, j);
3044 vars_op_array[j] = op_array;
3045 if (ssa->vars[j].use_chain >= 0
3046 && ssa->vars[j].alias == NO_ALIAS
3047 && zend_jit_var_supports_reg(ssa, j)) {
3048 start[j] = idx;
3049 flags[j] = ZREG_LOAD;
3050 count++;
3051 }
3052 j++;
3053 }
3054 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3055 SET_STACK_VAR(stack, i, -1);
3056 }
3057 level++;
3058 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3059 /* Close exiting call frames */
3060 for (i = 0; i < op_array->last_var; i++) {
3061 zend_jit_close_var(stack, i, start, end, flags, idx-1);
3062 }
3063 op_array = p->op_array;
3064 jit_extension =
3065 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3066 op_array_ssa = &jit_extension->func_info.ssa;
3067 frame = zend_jit_trace_ret_frame(frame, op_array);
3068 stack = frame->stack;
3069 if (level == 0) {
3070 /* New return frames */
3071 frame->prev = NULL;
3072 frame->func = (const zend_function*)op_array;
3073 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3074 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3075 SET_STACK_VAR(stack, i, j);
3076 vars_op_array[j] = op_array;
3077 if (ssa->vars[j].use_chain >= 0
3078 && ssa->vars[j].alias == NO_ALIAS
3079 && zend_jit_var_supports_reg(ssa, j)) {
3080 start[j] = idx;
3081 flags[j] = ZREG_LOAD;
3082 count++;
3083 }
3084 j++;
3085 }
3086 } else {
3087 level--;
3088 }
3089 } else if (p->op == ZEND_JIT_TRACE_END) {
3090 break;
3091 }
3092 }
3093
3094 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3095 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3096 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3097 zend_ssa_phi *phi = ssa->blocks[1].phis;
3098
3099 while (phi) {
3100 i = phi->sources[1];
3101 if (start[i] >= 0 && !ssa->vars[phi->ssa_var].no_val) {
3102 end[i] = idx;
3103 flags[i] &= ~ZREG_LAST_USE;
3104 }
3105 phi = phi->next;
3106 }
3107
3108 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3109 for (i = 0; i < op_array->last_var; i++) {
3110 if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) {
3111 end[i] = idx;
3112 flags[i] &= ~ZREG_LAST_USE;
3113 } else {
3114 zend_jit_close_var(stack, i, start, end, flags, idx);
3115 }
3116 }
3117 }
3118 } else {
3119 last_idx = idx;
3120 for (i = 0; i < op_array->last_var; i++) {
3121 zend_jit_close_var(stack, i, start, end, flags, idx);
3122 }
3123 while (frame->prev) {
3124 frame = frame->prev;
3125 op_array = &frame->func->op_array;
3126 stack = frame->stack;
3127 for (i = 0; i < op_array->last_var; i++) {
3128 zend_jit_close_var(stack, i, start, end, flags, idx);
3129 }
3130 }
3131 }
3132
3133 if (!count) {
3134 free_alloca(start, use_heap);
3135 zend_arena_release(&CG(arena), checkpoint);
3136 return NULL;
3137 }
3138
3139 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval));
3140 memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count);
3141 list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count);
3142 j = 0;
3143 for (i = 0; i < ssa->vars_count; i++) {
3144 if (start[i] >= 0 && end[i] >= 0) {
3145 ZEND_ASSERT(j < count);
3146 if ((flags[i] & ZREG_LOAD) &&
3147 (flags[i] & ZREG_LAST_USE) &&
3148 end[i] == ssa->vars[i].use_chain) {
3149 /* skip life range with single use */
3150 continue;
3151 }
3152 intervals[i] = &list[j];
3153 list[j].ssa_var = i;
3154 list[j].reg = ZREG_NONE;
3155 list[j].flags = flags[i];
3156 list[j].range.start = start[i];
3157 list[j].range.end = end[i];
3158 list[j].range.next = NULL;
3159 list[j].hint = NULL;
3160 list[j].used_as_hint = NULL;
3161 list[j].list_next = NULL;
3162 j++;
3163 }
3164 }
3165 count = j;
3166 free_alloca(start, use_heap);
3167 start = end = NULL;
3168
3169 if (!count) {
3170 zend_arena_release(&CG(arena), checkpoint);
3171 return NULL;
3172 }
3173
3174 /* Add hints */
3175 if (parent_vars_count) {
3176 i = trace_buffer->op_array->last_var;
3177 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3178 i += trace_buffer->op_array->T;
3179 }
3180 if ((uint32_t)i > parent_vars_count) {
3181 i = parent_vars_count;
3182 }
3183 while (i > 0) {
3184 i--;
3185 if (intervals[i]
3186 && STACK_REG(parent_stack, i) != ZREG_NONE
3187 && STACK_REG(parent_stack, i) < ZREG_NUM) {
3188 list[j].ssa_var = - 1;
3189 list[j].reg = STACK_REG(parent_stack, i);
3190 list[j].flags = 0;
3191 list[j].range.start = -1;
3192 list[j].range.end = -1;
3193 list[j].range.next = NULL;
3194 list[j].hint = NULL;
3195 list[j].used_as_hint = NULL;
3196 list[j].list_next = NULL;
3197 intervals[i]->hint = &list[j];
3198 j++;
3199 }
3200 }
3201 }
3202
3203 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3204 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3205 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3206 zend_ssa_phi *phi = ssa->blocks[1].phis;
3207
3208 while (phi) {
3209 if (intervals[phi->ssa_var]) {
3210 if (intervals[phi->sources[1]]
3211 && (ssa->var_info[phi->sources[1]].type & MAY_BE_ANY) ==
3212 (ssa->var_info[phi->ssa_var].type & MAY_BE_ANY)) {
3213 intervals[phi->sources[1]]->hint = intervals[phi->ssa_var];
3214 }
3215 }
3216 phi = phi->next;
3217 }
3218 }
3219
3220 for (i = 0; i < ssa->vars_count; i++) {
3221 if (intervals[i] && !intervals[i]->hint) {
3222
3223 if (ssa->vars[i].definition >= 0) {
3224 uint32_t line = ssa->vars[i].definition;
3225 const zend_op *opline = ssa_opcodes[line];
3226
3227 switch (opline->opcode) {
3228 case ZEND_QM_ASSIGN:
3229 case ZEND_POST_INC:
3230 case ZEND_POST_DEC:
3231 if (ssa->ops[line].op1_use >= 0 &&
3232 intervals[ssa->ops[line].op1_use] &&
3233 (i == ssa->ops[line].op1_def ||
3234 (i == ssa->ops[line].result_def &&
3235 (ssa->ops[line].op1_def < 0 ||
3236 !intervals[ssa->ops[line].op1_def])))) {
3237 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3238 }
3239 break;
3240 case ZEND_SEND_VAR:
3241 if (opline->op2_type == IS_CONST) {
3242 /* Named parameters not supported in JIT */
3243 break;
3244 }
3245 case ZEND_PRE_INC:
3246 case ZEND_PRE_DEC:
3247 if (i == ssa->ops[line].op1_def &&
3248 ssa->ops[line].op1_use >= 0 &&
3249 intervals[ssa->ops[line].op1_use]) {
3250 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3251 }
3252 break;
3253 case ZEND_ASSIGN:
3254 if (ssa->ops[line].op2_use >= 0 &&
3255 intervals[ssa->ops[line].op2_use] &&
3256 (i == ssa->ops[line].op2_def ||
3257 (i == ssa->ops[line].op1_def &&
3258 (ssa->ops[line].op2_def < 0 ||
3259 !intervals[ssa->ops[line].op2_def])) ||
3260 (i == ssa->ops[line].result_def &&
3261 (ssa->ops[line].op2_def < 0 ||
3262 !intervals[ssa->ops[line].op2_def]) &&
3263 (ssa->ops[line].op1_def < 0 ||
3264 !intervals[ssa->ops[line].op1_def])))) {
3265 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3266 }
3267 break;
3268 case ZEND_SUB:
3269 case ZEND_ADD:
3270 case ZEND_MUL:
3271 case ZEND_BW_OR:
3272 case ZEND_BW_AND:
3273 case ZEND_BW_XOR:
3274 if (i == ssa->ops[line].result_def) {
3275 if (ssa->ops[line].op1_use >= 0 &&
3276 intervals[ssa->ops[line].op1_use] &&
3277 ssa->ops[line].op1_use_chain < 0 &&
3278 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
3279 (ssa->var_info[i].type & MAY_BE_ANY) ==
3280 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
3281
3282 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op1_use].definition_phi;
3283 if (phi &&
3284 intervals[phi->sources[1]] &&
3285 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op1_use]) {
3286 break;
3287 }
3288 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
3289 } else if (opline->opcode != ZEND_SUB &&
3290 ssa->ops[line].op2_use >= 0 &&
3291 intervals[ssa->ops[line].op2_use] &&
3292 ssa->ops[line].op2_use_chain < 0 &&
3293 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
3294 (ssa->var_info[i].type & MAY_BE_ANY) ==
3295 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
3296
3297 zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op2_use].definition_phi;
3298 if (phi &&
3299 intervals[phi->sources[1]] &&
3300 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op2_use]) {
3301 break;
3302 }
3303 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
3304 }
3305 }
3306 break;
3307 }
3308 }
3309 }
3310 }
3311
3312 list = zend_jit_sort_intervals(intervals, ssa->vars_count);
3313
3314 if (list) {
3315 ival = list;
3316 while (ival) {
3317 if (ival->hint) {
3318 ival->hint->used_as_hint = ival;
3319 }
3320 ival = ival->list_next;
3321 }
3322 }
3323
3324 if (list) {
3325 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3326 fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM);
3327 ival = list;
3328 while (ival) {
3329 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3330 ival = ival->list_next;
3331 }
3332 }
3333 }
3334
3335 /* Linear Scan Register Allocation (op_array is not actually used, only fn_flags matters) */
3336 list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list);
3337
3338 if (list) {
3339 zend_lifetime_interval *ival, *next;
3340
3341 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
3342 ival = list;
3343 count = 0;
3344 while (ival != NULL) {
3345 ZEND_ASSERT(ival->reg != ZREG_NONE);
3346 count++;
3347 next = ival->list_next;
3348 ival->list_next = intervals[ival->ssa_var];
3349 intervals[ival->ssa_var] = ival;
3350 ival = next;
3351 }
3352
3353 if (!count) {
3354 zend_arena_release(&CG(arena), checkpoint);
3355 return NULL;
3356 }
3357
3358 /* Add LOAD flag to registers that can't reuse register from parent trace */
3359 if (parent_vars_count) {
3360 i = trace_buffer->op_array->last_var;
3361 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
3362 i += trace_buffer->op_array->T;
3363 }
3364 if ((uint32_t)i > parent_vars_count) {
3365 i = parent_vars_count;
3366 }
3367 while (i > 0) {
3368 i--;
3369 if (intervals[i] && intervals[i]->reg != STACK_REG(parent_stack, i)) {
3370 intervals[i]->flags |= ZREG_LOAD;
3371 }
3372 }
3373 }
3374
3375 /* SSA resolution */
3376 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3377 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3378 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3379 zend_ssa_phi *phi = ssa->blocks[1].phis;
3380
3381 while (phi) {
3382 int def = phi->ssa_var;
3383 int use = phi->sources[1];
3384
3385 if (intervals[def]) {
3386 if (!intervals[use]) {
3387 intervals[def]->flags |= ZREG_LOAD;
3388 if ((intervals[def]->flags & ZREG_LAST_USE)
3389 && ssa->vars[def].use_chain >= 0
3390 && ssa->vars[def].use_chain == intervals[def]->range.end) {
3391 /* remove interval used once */
3392 intervals[def] = NULL;
3393 count--;
3394 }
3395 } else if (intervals[def]->reg != intervals[use]->reg) {
3396 intervals[def]->flags |= ZREG_LOAD;
3397 if (ssa->vars[use].use_chain >= 0) {
3398 intervals[use]->flags |= ZREG_STORE;
3399 } else {
3400 intervals[use] = NULL;
3401 count--;
3402 }
3403 } else {
3404 use = phi->sources[0];
3405 ZEND_ASSERT(!intervals[use]);
3406 intervals[use] = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
3407 intervals[use]->ssa_var = phi->sources[0];
3408 intervals[use]->reg = intervals[def]->reg;
3409 intervals[use]->flags = ZREG_LOAD;
3410 intervals[use]->range.start = 0;
3411 intervals[use]->range.end = 0;
3412 intervals[use]->range.next = NULL;
3413 intervals[use]->hint = NULL;
3414 intervals[use]->used_as_hint = NULL;
3415 intervals[use]->list_next = NULL;
3416 }
3417 } else if (intervals[use]
3418 && (!ssa->vars[def].no_val
3419 || ssa->var_info[def].type != ssa->var_info[use].type)) {
3420 if (ssa->vars[use].use_chain >= 0) {
3421 intervals[use]->flags |= ZREG_STORE;
3422 } else {
3423 intervals[use] = NULL;
3424 count--;
3425 }
3426 }
3427 phi = phi->next;
3428 }
3429 } else {
3430 for (i = 0; i < ssa->vars_count; i++) {
3431 if (intervals[i]
3432 && intervals[i]->range.end == last_idx
3433 && !(intervals[i]->flags & (ZREG_LOAD|ZREG_STORE))) {
3434 intervals[i]->flags |= ZREG_STORE;
3435 }
3436 }
3437 }
3438
3439 if (!count) {
3440 zend_arena_release(&CG(arena), checkpoint);
3441 return NULL;
3442 }
3443
3444 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3445 fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM);
3446 for (i = 0; i < ssa->vars_count; i++) {
3447 ival = intervals[i];
3448 while (ival) {
3449 zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival);
3450 ival = ival->list_next;
3451 }
3452 }
3453 }
3454
3455 return intervals;
3456 }
3457
3458 zend_arena_release(&CG(arena), checkpoint); //???
3459 return NULL;
3460 }
3461
zend_jit_trace_clenup_stack(zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,zend_lifetime_interval ** ra)3462 static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, zend_lifetime_interval **ra)
3463 {
3464 uint32_t line = ssa_op - ssa->ops;
3465
3466 if (ssa_op->op1_use >= 0
3467 && ra[ssa_op->op1_use]
3468 && ra[ssa_op->op1_use]->range.end == line) {
3469 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
3470 }
3471 if (ssa_op->op2_use >= 0
3472 && ra[ssa_op->op2_use]
3473 && ra[ssa_op->op2_use]->range.end == line) {
3474 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op2.var), ZREG_NONE);
3475 }
3476 if (ssa_op->result_use >= 0
3477 && ra[ssa_op->result_use]
3478 && ra[ssa_op->result_use]->range.end == line) {
3479 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE);
3480 }
3481 }
3482
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3483 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3484 {
3485 zend_op *next_opline = (zend_op*)(opline + 1);
3486
3487 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3488 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3489 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3490 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3491 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3492 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3493 }
3494 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3495 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3496 }
3497 }
3498
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3499 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3500 {
3501 int var = ssa_op->result_def;
3502 int i;
3503 int use = ssa->vars[var].use_chain;
3504 const zend_op *opline;
3505
3506 if (use < 0
3507 || ssa->vars[var].phi_use_chain
3508 || ssa->ops[use].op1_use != var
3509 || ssa->ops[use].op1_use_chain != -1) {
3510 return 0;
3511 }
3512
3513 opline = ssa_opcodes[use];
3514 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3515 return (opline->op2_type == IS_CONST &&
3516 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3517 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3518 if (!JIT_G(current_frame)
3519 || !JIT_G(current_frame)->call
3520 || !JIT_G(current_frame)->call->func
3521 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3522 return 0;
3523 }
3524 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3525 && opline->opcode != ZEND_FETCH_OBJ_IS
3526 && opline->opcode != ZEND_FETCH_OBJ_W
3527 && opline->opcode != ZEND_ASSIGN_OBJ
3528 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3529 && opline->opcode != ZEND_PRE_INC_OBJ
3530 && opline->opcode != ZEND_PRE_DEC_OBJ
3531 && opline->opcode != ZEND_POST_INC_OBJ
3532 && opline->opcode != ZEND_POST_DEC_OBJ) {
3533 return 0;
3534 }
3535
3536 if (opline->op2_type != IS_CONST
3537 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3538 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3539 return 0;
3540 }
3541
3542 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3543 if (opline->op1_type == IS_CV
3544 && (opline+1)->op1_type == IS_CV
3545 && (opline+1)->op1.var == opline->op1.var) {
3546 /* skip $a->prop += $a; */
3547 return 0;
3548 }
3549 if (!zend_jit_supported_binary_op(
3550 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3551 return 0;
3552 }
3553 }
3554
3555 for (i = ssa->vars[var].definition; i < use; i++) {
3556 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3557 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3558 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3559 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3560 return 0;
3561 }
3562 }
3563
3564 return 1;
3565 }
3566
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3567 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3568 {
3569 uint32_t i;
3570
3571 for (i = 0; i < stack_size; i++) {
3572 if (STACK_REG(stack, i) != ZREG_NONE
3573 && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
3574 return 1;
3575 }
3576 }
3577 return 0;
3578 }
3579
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3580 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3581 {
3582 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3583 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3584 uint32_t stack_size;
3585 zend_jit_trace_stack *stack;
3586
3587 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3588 return 1;
3589 }
3590
3591 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3592 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3593 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3594 }
3595
zend_jit_trace_deoptimization(dasm_State ** Dst,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_lifetime_interval ** ra,bool polymorphic_side_trace)3596 static int zend_jit_trace_deoptimization(dasm_State **Dst,
3597 uint32_t flags,
3598 const zend_op *opline,
3599 zend_jit_trace_stack *parent_stack,
3600 int parent_vars_count,
3601 zend_ssa *ssa,
3602 zend_jit_trace_stack *stack,
3603 zend_lifetime_interval **ra,
3604 bool polymorphic_side_trace)
3605 {
3606 int i;
3607 bool has_constants = 0;
3608 bool has_unsaved_vars = 0;
3609
3610 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3611 for (i = 0; i < parent_vars_count; i++) {
3612 int8_t reg = STACK_REG(parent_stack, i);
3613
3614 if (reg != ZREG_NONE) {
3615 if (reg < ZREG_NUM) {
3616 if (ssa && ssa->vars[i].no_val) {
3617 /* pass */
3618 } else if (ra && ra[i] && ra[i]->reg == reg) {
3619 /* register already loaded by parent trace */
3620 if (stack) {
3621 SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i));
3622 }
3623 has_unsaved_vars = 1;
3624 } else {
3625 uint8_t type = STACK_TYPE(parent_stack, i);
3626
3627 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3628 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3629 STACK_MEM_TYPE(parent_stack, i) != type)) {
3630 return 0;
3631 }
3632 if (stack) {
3633 SET_STACK_TYPE(stack, i, type, 1);
3634 }
3635 }
3636 } else {
3637 /* delay custom deoptimization instructions to prevent register clobbering */
3638 has_constants = 1;
3639 }
3640 }
3641 }
3642
3643 if (has_unsaved_vars
3644 && (has_constants
3645 || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) {
3646 for (i = 0; i < parent_vars_count; i++) {
3647 int8_t reg = STACK_REG(parent_stack, i);
3648
3649 if (reg != ZREG_NONE) {
3650 if (reg < ZREG_NUM) {
3651 if (ssa && ssa->vars[i].no_val) {
3652 /* pass */
3653 } else if (ra && ra[i] && ra[i]->reg == reg) {
3654 uint8_t type = STACK_TYPE(parent_stack, i);
3655
3656 if (stack) {
3657 SET_STACK_TYPE(stack, i, type, 1);
3658 }
3659 if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE))
3660 && !zend_jit_store_var(Dst, 1 << type, i, reg,
3661 STACK_MEM_TYPE(parent_stack, i) != type)) {
3662 return 0;
3663 }
3664 }
3665 }
3666 }
3667 }
3668 }
3669
3670 if (has_constants) {
3671 for (i = 0; i < parent_vars_count; i++) {
3672 int8_t reg = STACK_REG(parent_stack, i);
3673
3674 if (reg != ZREG_NONE) {
3675 if (reg < ZREG_NUM) {
3676 /* pass */
3677 } else if (reg == ZREG_THIS) {
3678 if (polymorphic_side_trace) {
3679 ssa->var_info[i].delayed_fetch_this = 1;
3680 if (stack) {
3681 SET_STACK_REG(stack, i, ZREG_THIS);
3682 }
3683 } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
3684 return 0;
3685 }
3686 } else {
3687 if (reg == ZREG_ZVAL_COPY_GPR0
3688 &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) {
3689 return 0;
3690 }
3691 if (!zend_jit_store_const(Dst, i, reg)) {
3692 return 0;
3693 }
3694 }
3695 }
3696 }
3697 }
3698
3699 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3700 if (!zend_jit_save_call_chain(Dst, -1)) {
3701 return 0;
3702 }
3703 }
3704
3705 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3706 const zend_op *op = opline - 1;
3707
3708 if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) {
3709 return 0;
3710 }
3711 }
3712
3713 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3714 const zend_op *op = opline - 1;
3715
3716 if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) {
3717 return 0;
3718 }
3719 }
3720
3721 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3722 if (!zend_jit_check_exception(Dst)) {
3723 return 0;
3724 }
3725 }
3726
3727 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3728 if (!zend_jit_free_trampoline(Dst)) {
3729 return 0;
3730 }
3731 }
3732
3733 return 1;
3734 }
3735
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3736 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3737 {
3738 info->has_range = 1;
3739 info->range.min = min;
3740 info->range.max = max;
3741 info->range.underflow = 0;
3742 info->range.overflow = 0;
3743 }
3744
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3745 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3746 {
3747 zend_long op1_min, op1_max, op2_min, op2_max;
3748
3749 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3750 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3751 return;
3752 }
3753
3754 op1_min = OP1_MIN_RANGE();
3755 op1_max = OP1_MAX_RANGE();
3756 op2_min = OP2_MIN_RANGE();
3757 op2_max = OP2_MAX_RANGE();
3758
3759 switch (opline->opcode) {
3760 case ZEND_IS_EQUAL:
3761 case ZEND_CASE:
3762 case ZEND_IS_IDENTICAL:
3763 case ZEND_CASE_STRICT:
3764 case ZEND_IS_NOT_IDENTICAL:
3765 if (!exit_if_true) {
3766 /* op1 == op2 */
3767 if (ssa_op->op1_use >= 0) {
3768 zend_jit_trace_set_var_range(
3769 &ssa->var_info[ssa_op->op1_use],
3770 MAX(op1_min, op2_min),
3771 MIN(op1_max, op2_max));
3772 }
3773 if (ssa_op->op2_use >= 0) {
3774 zend_jit_trace_set_var_range(
3775 &ssa->var_info[ssa_op->op2_use],
3776 MAX(op2_min, op1_min),
3777 MIN(op2_max, op1_max));
3778 }
3779 }
3780 break;
3781 case ZEND_IS_NOT_EQUAL:
3782 if (exit_if_true) {
3783 /* op1 == op2 */
3784 if (ssa_op->op1_use >= 0) {
3785 zend_jit_trace_set_var_range(
3786 &ssa->var_info[ssa_op->op1_use],
3787 MAX(op1_min, op2_min),
3788 MIN(op1_max, op2_max));
3789 }
3790 if (ssa_op->op2_use >= 0) {
3791 zend_jit_trace_set_var_range(
3792 &ssa->var_info[ssa_op->op2_use],
3793 MAX(op2_min, op1_min),
3794 MIN(op2_max, op1_max));
3795 }
3796 }
3797 break;
3798 case ZEND_IS_SMALLER_OR_EQUAL:
3799 if (!exit_if_true) {
3800 /* op1 <= op2 */
3801 if (ssa_op->op1_use >= 0) {
3802 zend_jit_trace_set_var_range(
3803 &ssa->var_info[ssa_op->op1_use],
3804 op1_min,
3805 MIN(op1_max, op2_max));
3806 }
3807 if (ssa_op->op2_use >= 0) {
3808 zend_jit_trace_set_var_range(
3809 &ssa->var_info[ssa_op->op2_use],
3810 MAX(op2_min, op1_min),
3811 op2_max);
3812 }
3813 } else {
3814 /* op1 > op2 */
3815 if (ssa_op->op1_use >= 0) {
3816 zend_jit_trace_set_var_range(
3817 &ssa->var_info[ssa_op->op1_use],
3818 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3819 op1_max);
3820 }
3821 if (ssa_op->op2_use >= 0) {
3822 zend_jit_trace_set_var_range(
3823 &ssa->var_info[ssa_op->op2_use],
3824 op2_min,
3825 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3826 }
3827 }
3828 break;
3829 case ZEND_IS_SMALLER:
3830 if (!exit_if_true) {
3831 /* op1 < op2 */
3832 if (ssa_op->op1_use >= 0) {
3833 zend_jit_trace_set_var_range(
3834 &ssa->var_info[ssa_op->op1_use],
3835 op1_min,
3836 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3837 }
3838 if (ssa_op->op2_use >= 0) {
3839 zend_jit_trace_set_var_range(
3840 &ssa->var_info[ssa_op->op2_use],
3841 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3842 op2_max);
3843 }
3844 } else {
3845 /* op1 >= op2 */
3846 if (ssa_op->op1_use >= 0) {
3847 zend_jit_trace_set_var_range(
3848 &ssa->var_info[ssa_op->op1_use],
3849 MAX(op1_min, op2_min),
3850 op1_max);
3851 }
3852 if (ssa_op->op2_use >= 0) {
3853 zend_jit_trace_set_var_range(
3854 &ssa->var_info[ssa_op->op2_use],
3855 op2_min,
3856 MIN(op2_max, op1_max));
3857 }
3858 }
3859 break;
3860 }
3861 }
3862
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3863 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3864 {
3865 uint8_t prev_opcode;
3866
3867 if (opline->op1_type == IS_CONST
3868 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3869 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3870 if (ssa_op->op2_use >= 0) {
3871 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3872 ssa_op--;
3873 opline = ssa_opcodes[ssa_op - ssa->ops];
3874 prev_opcode = opline->opcode;
3875 if (prev_opcode == ZEND_PRE_INC
3876 || prev_opcode == ZEND_PRE_DEC
3877 || prev_opcode == ZEND_POST_INC
3878 || prev_opcode == ZEND_POST_DEC) {
3879 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3880 }
3881 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3882 ssa_op--;
3883 opline = ssa_opcodes[ssa_op - ssa->ops];
3884 prev_opcode = opline->opcode;
3885 if (prev_opcode == ZEND_ADD
3886 || prev_opcode == ZEND_SUB) {
3887 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3888 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3889 }
3890 }
3891 }
3892 } else if (opline->op2_type == IS_CONST
3893 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3894 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3895 if (ssa_op->op1_use >= 0) {
3896 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3897 ssa_op--;
3898 opline = ssa_opcodes[ssa_op - ssa->ops];
3899 prev_opcode = opline->opcode;
3900 if (prev_opcode == ZEND_PRE_INC
3901 || prev_opcode == ZEND_PRE_DEC
3902 || prev_opcode == ZEND_POST_INC
3903 || prev_opcode == ZEND_POST_DEC) {
3904 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3905 }
3906 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3907 ssa_op--;
3908 opline = ssa_opcodes[ssa_op - ssa->ops];
3909 prev_opcode = opline->opcode;
3910 if (prev_opcode == ZEND_ADD
3911 || prev_opcode == ZEND_SUB) {
3912 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3913 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3914 }
3915 }
3916 }
3917 } else {
3918 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3919 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3920
3921 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3922 && prev_ssa_op != ssa->ops
3923 && prev_ssa_op->op1_use >= 0
3924 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3925 prev_ssa_op--;
3926 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3927 }
3928
3929 if (ssa_op->op1_use == prev_ssa_op->op1_use
3930 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3931 if (prev_opcode == ZEND_IS_EQUAL
3932 || prev_opcode == ZEND_IS_NOT_EQUAL
3933 || prev_opcode == ZEND_IS_SMALLER
3934 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3935 || prev_opcode == ZEND_CASE
3936 || prev_opcode == ZEND_IS_IDENTICAL
3937 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3938 || prev_opcode == ZEND_CASE_STRICT) {
3939 if (ssa_op->op1_use < 0) {
3940 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3941 return 0;
3942 }
3943 }
3944 if (ssa_op->op2_use < 0) {
3945 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3946 return 0;
3947 }
3948 }
3949 return 1;
3950 }
3951 }
3952 }
3953 return 0;
3954 }
3955
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3956 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3957 zend_jit_trace_rec *p,
3958 zend_jit_trace_stack_frame *frame)
3959 {
3960 if (opline->result_type == IS_TMP_VAR
3961 && (p+1)->op == ZEND_JIT_TRACE_VM
3962 && (p+1)->opline == opline + 1
3963 && ((opline+1)->opcode == ZEND_SEND_VAL
3964 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3965 && frame
3966 && frame->call
3967 && frame->call->func
3968 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3969 && (opline+1)->op1_type == IS_TMP_VAR
3970 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3971 && (opline+1)->op1.var == opline->result.var) {
3972
3973 if (frame->call && frame->call->func) {
3974 uint8_t res_type = (p+1)->op1_type;
3975
3976 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3977 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3978 }
3979 }
3980 return 1;
3981 }
3982 return 0;
3983 }
3984
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)3985 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
3986 {
3987 const void *handler = NULL;
3988 dasm_State* dasm_state = NULL;
3989 zend_script *script = NULL;
3990 zend_lifetime_interval **ra = NULL;
3991 zend_string *name = NULL;
3992 void *checkpoint;
3993 const zend_op_array *op_array;
3994 zend_ssa *ssa, *op_array_ssa;
3995 const zend_op **ssa_opcodes;
3996 zend_jit_trace_rec *p;
3997 zend_jit_op_array_trace_extension *jit_extension;
3998 int num_op_arrays = 0;
3999 zend_jit_trace_info *t;
4000 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
4001 uint8_t smart_branch_opcode;
4002 const void *exit_addr;
4003 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info;
4004 bool send_result = 0;
4005 bool skip_comparison;
4006 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
4007 zend_class_entry *ce;
4008 bool ce_is_instanceof;
4009 bool on_this = 0;
4010 bool delayed_fetch_this = 0;
4011 bool avoid_refcounting = 0;
4012 bool polymorphic_side_trace =
4013 parent_trace &&
4014 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
4015 uint32_t i;
4016 zend_jit_trace_stack_frame *frame, *top, *call;
4017 zend_jit_trace_stack *stack;
4018 uint8_t res_type = IS_UNKNOWN;
4019 const zend_op *opline, *orig_opline;
4020 const zend_ssa_op *ssa_op, *orig_ssa_op;
4021 int checked_stack;
4022 int peek_checked_stack;
4023 uint32_t frame_flags = 0;
4024
4025 JIT_G(current_trace) = trace_buffer;
4026
4027 checkpoint = zend_arena_checkpoint(CG(arena));
4028
4029 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4030
4031 if (!ssa) {
4032 goto jit_cleanup;
4033 }
4034
4035 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4036
4037 /* Register allocation */
4038 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4039 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4040 ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4041 }
4042
4043 p = trace_buffer;
4044 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4045 op_array = p->op_array;
4046 frame = JIT_G(current_frame);
4047 top = zend_jit_trace_call_frame(frame, op_array);
4048 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4049 frame->used_stack = checked_stack = peek_checked_stack = 0;
4050 stack = frame->stack;
4051 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4052 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4053 }
4054
4055 opline = p[1].opline;
4056 name = zend_jit_trace_name(op_array, opline->lineno);
4057 p += ZEND_JIT_TRACE_START_REC_SIZE;
4058
4059 dasm_init(&dasm_state, DASM_MAXSECTION);
4060 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4061 dasm_setup(&dasm_state, dasm_actions);
4062
4063 jit_extension =
4064 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4065 op_array_ssa = &jit_extension->func_info.ssa;
4066
4067 dasm_growpc(&dasm_state, 2); /* =>0: loop header */
4068 /* =>1: end of code */
4069
4070 zend_jit_align_func(&dasm_state);
4071 if (!parent_trace) {
4072 zend_jit_prologue(&dasm_state);
4073 }
4074 zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM,
4075 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4076
4077 if (!parent_trace) {
4078 zend_jit_set_last_valid_opline(opline);
4079 zend_jit_track_last_valid_opline();
4080 } else {
4081 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4082 zend_jit_trace_opline_guard(&dasm_state, opline);
4083 } else {
4084 zend_jit_reset_last_valid_opline();
4085 }
4086 }
4087
4088 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4089 int last_var;
4090 int parent_vars_count = 0;
4091 zend_jit_trace_stack *parent_stack = NULL;
4092 int used_stack = ((zend_tssa*)ssa)->used_stack;
4093
4094 if (used_stack > 0) {
4095 peek_checked_stack = used_stack;
4096 if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) {
4097 goto jit_failure;
4098 }
4099 }
4100
4101 if (parent_trace) {
4102 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4103 op_array->last_var + op_array->T);
4104 if (parent_vars_count) {
4105 parent_stack =
4106 zend_jit_traces[parent_trace].stack_map +
4107 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4108 }
4109 }
4110
4111 last_var = op_array->last_var;
4112 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4113 last_var += op_array->T;
4114 }
4115
4116 for (i = 0; i < last_var; i++) {
4117 uint32_t info = ssa->var_info[i].type;
4118
4119 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4120 uint8_t type, mem_type;
4121
4122 type = concrete_type(info);
4123 if (i < parent_vars_count
4124 && STACK_TYPE(parent_stack, i) == type) {
4125 mem_type = STACK_MEM_TYPE(parent_stack, i);
4126 if (mem_type != IS_UNKNOWN) {
4127 SET_STACK_TYPE(stack, i, mem_type, 1);
4128 }
4129 SET_STACK_TYPE(stack, i, type, 0);
4130 } else {
4131 SET_STACK_TYPE(stack, i, type, 1);
4132 }
4133 } else if (ssa->vars[i].alias != NO_ALIAS) {
4134 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4135 } else if (i < parent_vars_count
4136 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4137 /* This must be already handled by trace type inference */
4138 ZEND_UNREACHABLE();
4139 // SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
4140 } else if ((info & MAY_BE_GUARD) != 0
4141 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4142 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4143 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4144 && (opline-1)->result_type == IS_VAR
4145 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4146 && (ssa->vars[i].use_chain != -1
4147 || (ssa->vars[i].phi_use_chain
4148 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4149 /* Check loop-invariant variable type */
4150 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4151 goto jit_failure;
4152 }
4153 info &= ~MAY_BE_GUARD;
4154 ssa->var_info[i].type = info;
4155 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4156 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4157 && op_array->function_name
4158 && i >= op_array->num_args) {
4159 /* This must be already handled by trace type inference */
4160 ZEND_UNREACHABLE();
4161 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4162 }
4163
4164 if ((info & MAY_BE_PACKED_GUARD) != 0
4165 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4166 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4167 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4168 && (ssa->vars[i].use_chain != -1
4169 || (ssa->vars[i].phi_use_chain
4170 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4171 if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) {
4172 goto jit_failure;
4173 }
4174 info &= ~MAY_BE_PACKED_GUARD;
4175 ssa->var_info[i].type = info;
4176 }
4177 }
4178
4179 if (parent_trace) {
4180 /* Deoptimization */
4181 if (!zend_jit_trace_deoptimization(&dasm_state,
4182 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4183 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4184 parent_stack, parent_vars_count, ssa, stack, ra,
4185 polymorphic_side_trace)) {
4186 goto jit_failure;
4187 }
4188 }
4189
4190 if (ra
4191 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4192 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4193 for (i = 0; i < last_var; i++) {
4194 if (ra[i]
4195 && (ra[i]->flags & ZREG_LOAD) != 0
4196 && ra[i]->reg != stack[i].reg) {
4197
4198 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4199 uint8_t op_type;
4200
4201 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4202 op_type = concrete_type(ssa->var_info[i].type);
4203 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
4204 goto jit_failure;
4205 }
4206 SET_STACK_TYPE(stack, i, op_type, 1);
4207 }
4208
4209 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4210 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4211 goto jit_failure;
4212 }
4213 }
4214 }
4215 }
4216 }
4217
4218 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4219 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4220 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4221
4222 zend_jit_label(&dasm_state, 0); /* start of of trace loop */
4223
4224 if (ra) {
4225 zend_ssa_phi *phi = ssa->blocks[1].phis;
4226
4227 while (phi) {
4228 zend_lifetime_interval *ival = ra[phi->ssa_var];
4229
4230 if (ival) {
4231 if (ival->flags & ZREG_LOAD) {
4232 uint32_t info = ssa->var_info[phi->ssa_var].type;
4233 ZEND_ASSERT(ival->reg != ZREG_NONE);
4234
4235 if (info & MAY_BE_GUARD) {
4236 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4237 goto jit_failure;
4238 }
4239 info &= ~MAY_BE_GUARD;
4240 ssa->var_info[phi->ssa_var].type = info;
4241 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4242 }
4243 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_LOAD);
4244 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
4245 goto jit_failure;
4246 }
4247 } else if (ival->flags & ZREG_STORE) {
4248 ZEND_ASSERT(ival->reg != ZREG_NONE);
4249
4250 SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_STORE);
4251 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg,
4252 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4253 goto jit_failure;
4254 }
4255 } else {
4256 /* Register has to be written back on side exit */
4257 SET_STACK_REG(stack, phi->var, ival->reg);
4258 }
4259 }
4260 phi = phi->next;
4261 }
4262 }
4263
4264 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4265 // if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4266 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4267 //
4268 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4269 // if (!timeout_exit_addr) {
4270 // goto jit_failure;
4271 // }
4272 // }
4273 // }
4274
4275 if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
4276 int last_var = op_array->last_var;
4277
4278 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4279 last_var += op_array->T;
4280 }
4281 for (i = 0; i < last_var; i++) {
4282 if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) {
4283 SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD);
4284 if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) {
4285 goto jit_failure;
4286 }
4287 }
4288 }
4289 }
4290 }
4291
4292 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4293 for (;;p++) {
4294 if (p->op == ZEND_JIT_TRACE_VM) {
4295 uint8_t op1_type = p->op1_type;
4296 uint8_t op2_type = p->op2_type;
4297 uint8_t op3_type = p->op3_type;
4298 uint8_t orig_op1_type = op1_type;
4299 uint8_t orig_op2_type = op2_type;
4300 uint8_t val_type = IS_UNKNOWN;
4301 bool op1_indirect;
4302 zend_class_entry *op1_ce = NULL;
4303 zend_class_entry *op2_ce = NULL;
4304 bool gen_handler = false;
4305
4306 opline = p->opline;
4307 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4308 op1_type = IS_UNKNOWN;
4309 }
4310 if (op1_type != IS_UNKNOWN) {
4311 op1_type &= ~IS_TRACE_PACKED;
4312 }
4313 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4314 op2_type = IS_UNKNOWN;
4315 }
4316 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4317 op3_type = IS_UNKNOWN;
4318 }
4319
4320 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4321 op1_ce = (zend_class_entry*)(p+1)->ce;
4322 p++;
4323 }
4324 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4325 op2_ce = (zend_class_entry*)(p+1)->ce;
4326 p++;
4327 }
4328 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4329 val_type = (p+1)->op1_type;
4330 p++;
4331 }
4332
4333 frame_flags = 0;
4334
4335 switch (opline->opcode) {
4336 case ZEND_INIT_FCALL:
4337 case ZEND_INIT_FCALL_BY_NAME:
4338 case ZEND_INIT_NS_FCALL_BY_NAME:
4339 case ZEND_INIT_METHOD_CALL:
4340 case ZEND_INIT_DYNAMIC_CALL:
4341 case ZEND_INIT_STATIC_METHOD_CALL:
4342 case ZEND_INIT_USER_CALL:
4343 case ZEND_NEW:
4344 frame->call_level++;
4345 }
4346
4347 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4348 gen_handler = 0;
4349 switch (opline->opcode) {
4350 case ZEND_PRE_INC:
4351 case ZEND_PRE_DEC:
4352 case ZEND_POST_INC:
4353 case ZEND_POST_DEC:
4354 if (opline->op1_type != IS_CV) {
4355 break;
4356 }
4357 op1_info = OP1_INFO();
4358 CHECK_OP1_TRACE_TYPE();
4359 if (!(op1_info & MAY_BE_LONG)) {
4360 break;
4361 }
4362 if (opline->result_type != IS_UNUSED) {
4363 res_use_info = zend_jit_trace_type_to_info(
4364 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4365 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4366 res_info = RES_INFO();
4367 res_addr = RES_REG_ADDR();
4368 } else {
4369 res_use_info = -1;
4370 res_info = -1;
4371 res_addr = 0;
4372 }
4373 op1_def_info = OP1_DEF_INFO();
4374 if (op1_def_info & MAY_BE_GUARD
4375 && !has_concrete_type(op1_def_info)) {
4376 op1_def_info &= ~MAY_BE_GUARD;
4377 }
4378 if (!zend_jit_inc_dec(&dasm_state, opline,
4379 op1_info, OP1_REG_ADDR(),
4380 op1_def_info, OP1_DEF_REG_ADDR(),
4381 res_use_info, res_info,
4382 res_addr,
4383 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4384 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4385 goto jit_failure;
4386 }
4387 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4388 && !(op1_info & MAY_BE_STRING)) {
4389 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4390 if (opline->result_type != IS_UNUSED) {
4391 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4392 }
4393 }
4394 if (opline->result_type != IS_UNUSED
4395 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4396 && !(op1_info & MAY_BE_STRING)) {
4397 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4398 }
4399 goto done;
4400 case ZEND_BW_OR:
4401 case ZEND_BW_AND:
4402 case ZEND_BW_XOR:
4403 case ZEND_SL:
4404 case ZEND_SR:
4405 case ZEND_MOD:
4406 op1_info = OP1_INFO();
4407 CHECK_OP1_TRACE_TYPE();
4408 op2_info = OP2_INFO();
4409 CHECK_OP2_TRACE_TYPE();
4410 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4411 break;
4412 }
4413 if (!(op1_info & MAY_BE_LONG)
4414 || !(op2_info & MAY_BE_LONG)) {
4415 break;
4416 }
4417 res_addr = RES_REG_ADDR();
4418 if (Z_MODE(res_addr) != IS_REG
4419 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4420 send_result = 1;
4421 res_use_info = -1;
4422 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4423 if (!zend_jit_reuse_ip(&dasm_state)) {
4424 goto jit_failure;
4425 }
4426 } else {
4427 res_use_info = zend_jit_trace_type_to_info(
4428 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4429 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4430 }
4431 res_info = RES_INFO();
4432 if (!zend_jit_long_math(&dasm_state, opline,
4433 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4434 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4435 res_use_info, res_info, res_addr,
4436 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4437 goto jit_failure;
4438 }
4439 goto done;
4440 case ZEND_ADD:
4441 case ZEND_SUB:
4442 case ZEND_MUL:
4443 // case ZEND_DIV: // TODO: check for division by zero ???
4444 op1_info = OP1_INFO();
4445 op1_addr = OP1_REG_ADDR();
4446 op2_info = OP2_INFO();
4447 op2_addr = OP2_REG_ADDR();
4448 if (orig_op1_type != IS_UNKNOWN
4449 && (orig_op1_type & IS_TRACE_REFERENCE)
4450 && opline->op1_type == IS_CV
4451 && (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_FCARG1)
4452 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4453 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4454 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4455 goto jit_failure;
4456 }
4457 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4458 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4459 }
4460 } else {
4461 CHECK_OP1_TRACE_TYPE();
4462 }
4463 if (orig_op2_type != IS_UNKNOWN
4464 && (orig_op2_type & IS_TRACE_REFERENCE)
4465 && opline->op2_type == IS_CV
4466 && (Z_MODE(op1_addr) != IS_REG || Z_REG(op1_addr) != ZREG_FCARG1)
4467 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4468 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op2_type, &op2_info, &op2_addr,
4469 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4470 goto jit_failure;
4471 }
4472 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4473 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4474 }
4475 } else {
4476 CHECK_OP2_TRACE_TYPE();
4477 }
4478 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4479 break;
4480 }
4481 if (opline->opcode == ZEND_ADD &&
4482 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4483 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4484 /* pass */
4485 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4486 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4487 break;
4488 }
4489 res_addr = RES_REG_ADDR();
4490 if (Z_MODE(res_addr) != IS_REG
4491 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4492 send_result = 1;
4493 res_use_info = -1;
4494 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4495 if (!zend_jit_reuse_ip(&dasm_state)) {
4496 goto jit_failure;
4497 }
4498 } else {
4499 res_use_info = zend_jit_trace_type_to_info(
4500 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)))
4501 & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4502 }
4503 res_info = RES_INFO();
4504 if (opline->opcode == ZEND_ADD &&
4505 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4506 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4507 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4508 goto jit_failure;
4509 }
4510 } else {
4511 if (!zend_jit_math(&dasm_state, opline,
4512 op1_info, op1_addr,
4513 op2_info, op2_addr,
4514 res_use_info, res_info, res_addr,
4515 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4516 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4517 goto jit_failure;
4518 }
4519 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4520 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4521 && has_concrete_type(op1_info)
4522 && has_concrete_type(op2_info)) {
4523 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4524 }
4525 }
4526 goto done;
4527 case ZEND_CONCAT:
4528 case ZEND_FAST_CONCAT:
4529 op1_info = OP1_INFO();
4530 CHECK_OP1_TRACE_TYPE();
4531 op2_info = OP2_INFO();
4532 CHECK_OP2_TRACE_TYPE();
4533 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4534 break;
4535 }
4536 if (!(op1_info & MAY_BE_STRING) ||
4537 !(op2_info & MAY_BE_STRING)) {
4538 break;
4539 }
4540 res_addr = RES_REG_ADDR();
4541 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4542 send_result = 1;
4543 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4544 if (!zend_jit_reuse_ip(&dasm_state)) {
4545 goto jit_failure;
4546 }
4547 }
4548 if (!zend_jit_concat(&dasm_state, opline,
4549 op1_info, op2_info, res_addr,
4550 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4551 goto jit_failure;
4552 }
4553 goto done;
4554 case ZEND_ASSIGN_OP:
4555 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4556 break;
4557 }
4558 op1_info = OP1_INFO();
4559 CHECK_OP1_TRACE_TYPE();
4560 op2_info = OP2_INFO();
4561 CHECK_OP2_TRACE_TYPE();
4562 if (!zend_jit_supported_binary_op(
4563 opline->extended_value, op1_info, op2_info)) {
4564 break;
4565 }
4566 op1_def_info = OP1_DEF_INFO();
4567 if (op1_def_info & MAY_BE_GUARD
4568 && !has_concrete_type(op1_def_info)) {
4569 op1_def_info &= ~MAY_BE_GUARD;
4570 }
4571 if (!zend_jit_assign_op(&dasm_state, opline,
4572 op1_info, op1_def_info, OP1_RANGE(),
4573 op2_info, OP2_RANGE(),
4574 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4575 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4576 goto jit_failure;
4577 }
4578 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4579 && has_concrete_type(op1_info)
4580 && has_concrete_type(op2_info)) {
4581 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4582 if (opline->result_type != IS_UNUSED) {
4583 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4584 }
4585 }
4586 goto done;
4587 case ZEND_ASSIGN_DIM_OP:
4588 if (opline->result_type != IS_UNUSED) {
4589 break;
4590 }
4591 if (!zend_jit_supported_binary_op(
4592 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4593 break;
4594 }
4595 if (opline->op1_type == IS_CV
4596 && (opline+1)->op1_type == IS_CV
4597 && (opline+1)->op1.var == opline->op1.var) {
4598 /* skip $a[x] += $a; */
4599 break;
4600 }
4601 op1_info = OP1_INFO();
4602 op1_addr = OP1_REG_ADDR();
4603 if (opline->op1_type == IS_VAR) {
4604 if (orig_op1_type != IS_UNKNOWN
4605 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4606 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4607 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4608 goto jit_failure;
4609 }
4610 } else {
4611 break;
4612 }
4613 }
4614 if (orig_op1_type != IS_UNKNOWN
4615 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4616 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4617 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4618 goto jit_failure;
4619 }
4620 if (opline->op1_type == IS_CV
4621 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4622 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4623 }
4624 } else {
4625 CHECK_OP1_TRACE_TYPE();
4626 }
4627 op2_info = OP2_INFO();
4628 CHECK_OP2_TRACE_TYPE();
4629 op1_data_info = OP1_DATA_INFO();
4630 CHECK_OP1_DATA_TRACE_TYPE();
4631 op1_def_info = OP1_DEF_INFO();
4632 if (!zend_jit_assign_dim_op(&dasm_state, opline,
4633 op1_info, op1_def_info, op1_addr, op2_info,
4634 op1_data_info, OP1_DATA_RANGE(), val_type,
4635 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4636 goto jit_failure;
4637 }
4638 goto done;
4639 case ZEND_PRE_INC_OBJ:
4640 case ZEND_PRE_DEC_OBJ:
4641 case ZEND_POST_INC_OBJ:
4642 case ZEND_POST_DEC_OBJ:
4643 if (opline->op2_type != IS_CONST
4644 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4645 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4646 break;
4647 }
4648 ce = NULL;
4649 ce_is_instanceof = 0;
4650 on_this = delayed_fetch_this = 0;
4651 op1_indirect = 0;
4652 if (opline->op1_type == IS_UNUSED) {
4653 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4654 ce = op_array->scope;
4655 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4656 op1_addr = 0;
4657 on_this = 1;
4658 } else {
4659 if (ssa_op->op1_use >= 0) {
4660 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4661 }
4662 op1_info = OP1_INFO();
4663 if (!(op1_info & MAY_BE_OBJECT)) {
4664 break;
4665 }
4666 op1_addr = OP1_REG_ADDR();
4667 if (opline->op1_type == IS_VAR) {
4668 if (orig_op1_type != IS_UNKNOWN
4669 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4670 op1_indirect = 1;
4671 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4672 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4673 goto jit_failure;
4674 }
4675 }
4676 }
4677 if (orig_op1_type != IS_UNKNOWN
4678 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4679 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4680 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4681 goto jit_failure;
4682 }
4683 if (opline->op1_type == IS_CV
4684 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4685 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4686 }
4687 } else {
4688 CHECK_OP1_TRACE_TYPE();
4689 }
4690 if (!(op1_info & MAY_BE_OBJECT)) {
4691 break;
4692 }
4693 if (ssa->var_info && ssa->ops) {
4694 if (ssa_op->op1_use >= 0) {
4695 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4696 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4697 ce = op1_ssa->ce;
4698 ce_is_instanceof = op1_ssa->is_instanceof;
4699 }
4700 }
4701 }
4702 if (delayed_fetch_this) {
4703 on_this = 1;
4704 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4705 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4706 } else if (op_array_ssa->ops
4707 && op_array_ssa->vars
4708 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4709 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4710 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4711 }
4712 }
4713 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4714 op1_info, op1_addr,
4715 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4716 val_type)) {
4717 goto jit_failure;
4718 }
4719 goto done;
4720 case ZEND_ASSIGN_OBJ_OP:
4721 if (opline->result_type != IS_UNUSED) {
4722 break;
4723 }
4724 if (opline->op2_type != IS_CONST
4725 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4726 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4727 break;
4728 }
4729 if (opline->op1_type == IS_CV
4730 && (opline+1)->op1_type == IS_CV
4731 && (opline+1)->op1.var == opline->op1.var) {
4732 /* skip $a->prop += $a; */
4733 break;
4734 }
4735 if (!zend_jit_supported_binary_op(
4736 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4737 break;
4738 }
4739 ce = NULL;
4740 ce_is_instanceof = 0;
4741 on_this = delayed_fetch_this = 0;
4742 op1_indirect = 0;
4743 if (opline->op1_type == IS_UNUSED) {
4744 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4745 ce = op_array->scope;
4746 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4747 op1_addr = 0;
4748 on_this = 1;
4749 } else {
4750 if (ssa_op->op1_use >= 0) {
4751 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4752 }
4753 op1_info = OP1_INFO();
4754 if (!(op1_info & MAY_BE_OBJECT)) {
4755 break;
4756 }
4757 op1_addr = OP1_REG_ADDR();
4758 if (opline->op1_type == IS_VAR) {
4759 if (orig_op1_type != IS_UNKNOWN
4760 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4761 op1_indirect = 1;
4762 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4763 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4764 goto jit_failure;
4765 }
4766 }
4767 }
4768 if (orig_op1_type != IS_UNKNOWN
4769 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4770 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4771 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4772 goto jit_failure;
4773 }
4774 if (opline->op1_type == IS_CV
4775 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4776 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4777 }
4778 } else {
4779 CHECK_OP1_TRACE_TYPE();
4780 }
4781 if (!(op1_info & MAY_BE_OBJECT)) {
4782 break;
4783 }
4784 if (ssa->var_info && ssa->ops) {
4785 if (ssa_op->op1_use >= 0) {
4786 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4787 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4788 ce = op1_ssa->ce;
4789 ce_is_instanceof = op1_ssa->is_instanceof;
4790 }
4791 }
4792 }
4793 if (delayed_fetch_this) {
4794 on_this = 1;
4795 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4796 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4797 } else if (op_array_ssa->ops
4798 && op_array_ssa->vars
4799 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4800 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4801 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4802 }
4803 }
4804 op1_data_info = OP1_DATA_INFO();
4805 CHECK_OP1_DATA_TRACE_TYPE();
4806 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
4807 op1_info, op1_addr, op1_data_info, OP1_DATA_RANGE(),
4808 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4809 val_type)) {
4810 goto jit_failure;
4811 }
4812 goto done;
4813 case ZEND_ASSIGN_OBJ:
4814 if (opline->op2_type != IS_CONST
4815 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4816 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4817 break;
4818 }
4819 ce = NULL;
4820 ce_is_instanceof = 0;
4821 on_this = delayed_fetch_this = 0;
4822 op1_indirect = 0;
4823 if (opline->op1_type == IS_UNUSED) {
4824 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4825 ce = op_array->scope;
4826 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4827 op1_addr = 0;
4828 on_this = 1;
4829 } else {
4830 if (ssa_op->op1_use >= 0) {
4831 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4832 }
4833 op1_info = OP1_INFO();
4834 if (!(op1_info & MAY_BE_OBJECT)) {
4835 break;
4836 }
4837 op1_addr = OP1_REG_ADDR();
4838 if (opline->op1_type == IS_VAR) {
4839 if (orig_op1_type != IS_UNKNOWN
4840 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4841 op1_indirect = 1;
4842 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4843 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4844 goto jit_failure;
4845 }
4846 }
4847 }
4848 if (orig_op1_type != IS_UNKNOWN
4849 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4850 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4851 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4852 goto jit_failure;
4853 }
4854 if (opline->op1_type == IS_CV
4855 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4856 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4857 }
4858 } else {
4859 CHECK_OP1_TRACE_TYPE();
4860 }
4861 if (!(op1_info & MAY_BE_OBJECT)) {
4862 break;
4863 }
4864 if (ssa->var_info && ssa->ops) {
4865 if (ssa_op->op1_use >= 0) {
4866 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4867 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4868 ce = op1_ssa->ce;
4869 ce_is_instanceof = op1_ssa->is_instanceof;
4870 }
4871 }
4872 }
4873 if (delayed_fetch_this) {
4874 on_this = 1;
4875 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4876 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4877 } else if (op_array_ssa->ops
4878 && op_array_ssa->vars
4879 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4880 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4881 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4882 }
4883 }
4884 op1_data_info = OP1_DATA_INFO();
4885 CHECK_OP1_DATA_TRACE_TYPE();
4886 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
4887 op1_info, op1_addr, op1_data_info,
4888 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4889 val_type,
4890 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4891 goto jit_failure;
4892 }
4893 if ((opline+1)->op1_type == IS_CV
4894 && (ssa_op+1)->op1_def >= 0
4895 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4896 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4897 }
4898 goto done;
4899 case ZEND_ASSIGN_DIM:
4900 op1_info = OP1_INFO();
4901 op1_addr = OP1_REG_ADDR();
4902 if (opline->op1_type == IS_CV
4903 && (opline+1)->op1_type == IS_CV
4904 && (opline+1)->op1.var == opline->op1.var) {
4905 /* skip $a[x] = $a; */
4906 break;
4907 }
4908 if (opline->op1_type == IS_VAR) {
4909 if (orig_op1_type != IS_UNKNOWN
4910 && (orig_op1_type & IS_TRACE_INDIRECT)
4911 && opline->result_type == IS_UNUSED) {
4912 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
4913 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4914 goto jit_failure;
4915 }
4916 } else {
4917 break;
4918 }
4919 }
4920 if (orig_op1_type != IS_UNKNOWN
4921 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4922 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4923 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4924 goto jit_failure;
4925 }
4926 if (opline->op1_type == IS_CV
4927 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4928 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4929 }
4930 } else {
4931 CHECK_OP1_TRACE_TYPE();
4932 }
4933 op2_info = OP2_INFO();
4934 CHECK_OP2_TRACE_TYPE();
4935 op1_data_info = OP1_DATA_INFO();
4936 CHECK_OP1_DATA_TRACE_TYPE();
4937 if (!zend_jit_assign_dim(&dasm_state, opline,
4938 op1_info, op1_addr, op2_info, op1_data_info, val_type,
4939 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4940 goto jit_failure;
4941 }
4942 if ((opline+1)->op1_type == IS_CV
4943 && (ssa_op+1)->op1_def >= 0
4944 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4945 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4946 }
4947 goto done;
4948 case ZEND_ASSIGN:
4949 if (opline->op1_type != IS_CV) {
4950 break;
4951 }
4952 op2_addr = OP2_REG_ADDR();
4953 op2_info = OP2_INFO();
4954
4955 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
4956 op2_def_addr = op2_addr;
4957 } else {
4958 op2_def_addr = OP2_DEF_REG_ADDR();
4959 }
4960 CHECK_OP2_TRACE_TYPE();
4961 op1_info = OP1_INFO();
4962 op1_def_info = OP1_DEF_INFO();
4963 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
4964 if (op1_type < IS_STRING
4965 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
4966 if (!zend_jit_scalar_type_guard(&dasm_state, opline, opline->op1.var)) {
4967 goto jit_failure;
4968 }
4969 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
4970 } else {
4971 CHECK_OP1_TRACE_TYPE();
4972 }
4973 }
4974 op1_addr = OP1_REG_ADDR();
4975 op1_def_addr = OP1_DEF_REG_ADDR();
4976 if (Z_MODE(op1_def_addr) != IS_REG &&
4977 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
4978 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
4979 /* type may be not set */
4980 op1_info |= MAY_BE_NULL;
4981 }
4982 if (orig_op1_type != IS_UNKNOWN) {
4983 if (orig_op1_type & IS_TRACE_REFERENCE) {
4984 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
4985 !ssa->var_info[ssa_op->op1_use].guarded_reference, 0)) {
4986 goto jit_failure;
4987 }
4988 if (opline->op1_type == IS_CV
4989 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4990 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4991 }
4992 if (opline->result_type == IS_UNUSED) {
4993 res_addr = 0;
4994 } else {
4995 res_addr = RES_REG_ADDR();
4996 if (Z_MODE(res_addr) != IS_REG
4997 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4998 send_result = 1;
4999 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5000 if (!zend_jit_reuse_ip(&dasm_state)) {
5001 goto jit_failure;
5002 }
5003 }
5004 }
5005 if (!zend_jit_assign_to_typed_ref(&dasm_state, opline, opline->op2_type, op2_addr, res_addr, 1)) {
5006 goto jit_failure;
5007 }
5008 op1_def_addr = op1_addr;
5009 op1_def_info &= ~MAY_BE_REF;
5010 } else if (op1_info & MAY_BE_REF) {
5011 if (!zend_jit_noref_guard(&dasm_state, opline, op1_addr)) {
5012 goto jit_failure;
5013 }
5014 op1_info &= ~MAY_BE_REF;
5015 op1_def_info &= ~MAY_BE_REF;
5016 }
5017 }
5018 if (opline->result_type == IS_UNUSED) {
5019 res_addr = 0;
5020 res_info = -1;
5021 } else {
5022 res_addr = RES_REG_ADDR();
5023 res_info = RES_INFO();
5024 if (Z_MODE(res_addr) != IS_REG
5025 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5026 send_result = 1;
5027 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5028 if (!zend_jit_reuse_ip(&dasm_state)) {
5029 goto jit_failure;
5030 }
5031 }
5032 }
5033 if (!zend_jit_assign(&dasm_state, opline,
5034 op1_info, op1_addr,
5035 op1_def_info, op1_def_addr,
5036 op2_info, op2_addr, op2_def_addr,
5037 res_info, res_addr,
5038 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5039 goto jit_failure;
5040 }
5041 if (ssa_op->op2_def >= 0
5042 && Z_MODE(op2_addr) == IS_REG
5043 && ssa->vars[ssa_op->op2_def].no_val) {
5044 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5045 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5046
5047 if (STACK_MEM_TYPE(stack, var_num) != type
5048 && ssa->vars[ssa_op->op2_def].use_chain < 0
5049 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5050 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5051 return 0;
5052 }
5053 SET_STACK_TYPE(stack, var_num, type, 1);
5054 }
5055 }
5056 if (opline->op2_type == IS_CV
5057 && ssa_op->op2_def >= 0
5058 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5059 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5060 }
5061 goto done;
5062 case ZEND_CAST:
5063 if (opline->extended_value != op1_type) {
5064 break;
5065 }
5066 ZEND_FALLTHROUGH;
5067 case ZEND_QM_ASSIGN:
5068 op1_addr = OP1_REG_ADDR();
5069 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5070 op1_def_addr = op1_addr;
5071 } else {
5072 op1_def_addr = OP1_DEF_REG_ADDR();
5073 }
5074 op1_info = OP1_INFO();
5075 CHECK_OP1_TRACE_TYPE();
5076 res_info = RES_INFO();
5077 res_use_info = zend_jit_trace_type_to_info(
5078 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
5079 res_addr = RES_REG_ADDR();
5080 if (Z_MODE(res_addr) != IS_REG &&
5081 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5082 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5083 /* type may be not set */
5084 res_use_info |= MAY_BE_NULL;
5085 }
5086 if (!zend_jit_qm_assign(&dasm_state, opline,
5087 op1_info, op1_addr, op1_def_addr,
5088 res_use_info, res_info, res_addr)) {
5089 goto jit_failure;
5090 }
5091 if (ssa_op->op1_def >= 0
5092 && Z_MODE(op1_addr) == IS_REG
5093 && ssa->vars[ssa_op->op1_def].no_val) {
5094 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5095 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5096
5097 if (STACK_MEM_TYPE(stack, var_num) != type
5098 && ssa->vars[ssa_op->op1_def].use_chain < 0
5099 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5100 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5101 return 0;
5102 }
5103 SET_STACK_TYPE(stack, var_num, type, 1);
5104 }
5105 }
5106 if (opline->op1_type == IS_CV
5107 && ssa_op->op1_def >= 0
5108 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5109 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5110 }
5111 goto done;
5112 case ZEND_INIT_FCALL:
5113 case ZEND_INIT_FCALL_BY_NAME:
5114 case ZEND_INIT_NS_FCALL_BY_NAME:
5115 frame_flags = TRACE_FRAME_MASK_NESTED;
5116 if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5117 goto jit_failure;
5118 }
5119 goto done;
5120 case ZEND_SEND_VAL:
5121 case ZEND_SEND_VAL_EX:
5122 if (opline->op2_type == IS_CONST) {
5123 /* Named parameters not supported in JIT */
5124 break;
5125 }
5126 if (opline->opcode == ZEND_SEND_VAL_EX
5127 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5128 break;
5129 }
5130 op1_info = OP1_INFO();
5131 CHECK_OP1_TRACE_TYPE();
5132 if (!zend_jit_send_val(&dasm_state, opline,
5133 op1_info, OP1_REG_ADDR())) {
5134 goto jit_failure;
5135 }
5136 if (frame->call && frame->call->func) {
5137 if (opline->op1_type == IS_CONST) {
5138 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5139 } else if (op1_type != IS_UNKNOWN) {
5140 if (op1_type == IS_UNDEF) {
5141 op1_type = IS_NULL;
5142 }
5143 zend_jit_trace_send_type(opline, frame->call, op1_type);
5144 }
5145 }
5146 goto done;
5147 case ZEND_SEND_REF:
5148 if (opline->op2_type == IS_CONST) {
5149 /* Named parameters not supported in JIT */
5150 break;
5151 }
5152 op1_info = OP1_INFO();
5153 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
5154 op1_info, 0)) {
5155 goto jit_failure;
5156 }
5157 if (opline->op1_type == IS_CV
5158 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5159 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5160 }
5161 goto done;
5162 case ZEND_SEND_VAR:
5163 case ZEND_SEND_VAR_EX:
5164 case ZEND_SEND_VAR_NO_REF:
5165 case ZEND_SEND_VAR_NO_REF_EX:
5166 case ZEND_SEND_FUNC_ARG:
5167 if (opline->op2_type == IS_CONST) {
5168 /* Named parameters not supported in JIT */
5169 break;
5170 }
5171 if ((opline->opcode == ZEND_SEND_VAR_EX
5172 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5173 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5174 break;
5175 }
5176 op1_addr = OP1_REG_ADDR();
5177 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5178 op1_def_addr = op1_addr;
5179 } else {
5180 op1_def_addr = OP1_DEF_REG_ADDR();
5181 }
5182 op1_info = OP1_INFO();
5183 CHECK_OP1_TRACE_TYPE();
5184 if (!zend_jit_send_var(&dasm_state, opline, op_array,
5185 op1_info, op1_addr, op1_def_addr)) {
5186 goto jit_failure;
5187 }
5188 if (ssa_op->op1_def >= 0
5189 && Z_MODE(op1_addr) == IS_REG
5190 && ssa->vars[ssa_op->op1_def].no_val) {
5191 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5192 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5193
5194 if (STACK_MEM_TYPE(stack, var_num) != type
5195 && ssa->vars[ssa_op->op1_def].use_chain < 0
5196 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5197 if (!zend_jit_store_var_type(&dasm_state, var_num, type)) {
5198 return 0;
5199 }
5200 SET_STACK_TYPE(stack, var_num, type, 1);
5201 }
5202 }
5203 if (opline->op1_type == IS_CV
5204 && ssa_op->op1_def >= 0
5205 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5206 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5207 }
5208 if (frame->call && frame->call->func) {
5209 if ((opline->opcode == ZEND_SEND_VAR_EX
5210 || opline->opcode == ZEND_SEND_FUNC_ARG)
5211 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5212 goto done;
5213 }
5214 if (op1_type != IS_UNKNOWN) {
5215 if (op1_type == IS_UNDEF) {
5216 op1_type = IS_NULL;
5217 }
5218 zend_jit_trace_send_type(opline, frame->call, op1_type);
5219 }
5220 }
5221 goto done;
5222 case ZEND_CHECK_FUNC_ARG:
5223 if (!JIT_G(current_frame)
5224 || !JIT_G(current_frame)->call
5225 || !JIT_G(current_frame)->call->func) {
5226 break;
5227 }
5228 if (opline->op2_type == IS_CONST
5229 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5230 /* Named parameters not supported in JIT */
5231 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5232 break;
5233 }
5234 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
5235 goto jit_failure;
5236 }
5237 goto done;
5238 case ZEND_CHECK_UNDEF_ARGS:
5239 if (JIT_G(current_frame)
5240 && JIT_G(current_frame)->call) {
5241 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5242 }
5243 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
5244 goto jit_failure;
5245 }
5246 goto done;
5247 case ZEND_DO_UCALL:
5248 case ZEND_DO_ICALL:
5249 case ZEND_DO_FCALL_BY_NAME:
5250 case ZEND_DO_FCALL:
5251 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5252 goto jit_failure;
5253 }
5254 goto done;
5255 case ZEND_IS_EQUAL:
5256 case ZEND_IS_NOT_EQUAL:
5257 case ZEND_IS_SMALLER:
5258 case ZEND_IS_SMALLER_OR_EQUAL:
5259 case ZEND_CASE:
5260 op1_info = OP1_INFO();
5261 op2_info = OP2_INFO();
5262 skip_comparison =
5263 ssa_op != ssa->ops &&
5264 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5265 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5266 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5267 CHECK_OP1_TRACE_TYPE();
5268 CHECK_OP2_TRACE_TYPE();
5269 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5270 bool exit_if_true = 0;
5271 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5272 uint32_t exit_point;
5273
5274 if (ra) {
5275 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5276 }
5277 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5278 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5279 if (!exit_addr) {
5280 goto jit_failure;
5281 }
5282 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5283 if (!zend_jit_cmp(&dasm_state, opline,
5284 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5285 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5286 RES_REG_ADDR(),
5287 zend_may_throw(opline, ssa_op, op_array, ssa),
5288 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5289 goto jit_failure;
5290 }
5291 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5292 } else {
5293 smart_branch_opcode = 0;
5294 exit_addr = NULL;
5295 if (!zend_jit_cmp(&dasm_state, opline,
5296 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5297 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5298 RES_REG_ADDR(),
5299 zend_may_throw(opline, ssa_op, op_array, ssa),
5300 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5301 goto jit_failure;
5302 }
5303 }
5304 goto done;
5305 case ZEND_IS_IDENTICAL:
5306 case ZEND_IS_NOT_IDENTICAL:
5307 case ZEND_CASE_STRICT:
5308 op1_info = OP1_INFO();
5309 op2_info = OP2_INFO();
5310 skip_comparison =
5311 ssa_op != ssa->ops &&
5312 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5313 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5314 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5315 CHECK_OP1_TRACE_TYPE();
5316 CHECK_OP2_TRACE_TYPE();
5317 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5318 bool exit_if_true = 0;
5319 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5320 uint32_t exit_point;
5321
5322 if (ra) {
5323 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5324 }
5325 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5326 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5327 if (!exit_addr) {
5328 goto jit_failure;
5329 }
5330 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5331 exit_if_true = !exit_if_true;
5332 }
5333 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5334 if (!zend_jit_identical(&dasm_state, opline,
5335 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5336 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5337 RES_REG_ADDR(),
5338 zend_may_throw(opline, ssa_op, op_array, ssa),
5339 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5340 goto jit_failure;
5341 }
5342 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5343 } else {
5344 smart_branch_opcode = 0;
5345 exit_addr = NULL;
5346 if (!zend_jit_identical(&dasm_state, opline,
5347 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5348 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5349 RES_REG_ADDR(),
5350 zend_may_throw(opline, ssa_op, op_array, ssa),
5351 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5352 goto jit_failure;
5353 }
5354 }
5355 goto done;
5356 case ZEND_DEFINED:
5357 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5358 bool exit_if_true = 0;
5359 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5360 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5361
5362 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5363 if (!exit_addr) {
5364 goto jit_failure;
5365 }
5366 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5367 } else {
5368 smart_branch_opcode = 0;
5369 exit_addr = NULL;
5370 }
5371 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5372 goto jit_failure;
5373 }
5374 goto done;
5375 case ZEND_TYPE_CHECK:
5376 if (opline->extended_value == MAY_BE_RESOURCE) {
5377 // TODO: support for is_resource() ???
5378 break;
5379 }
5380 op1_info = OP1_INFO();
5381 CHECK_OP1_TRACE_TYPE();
5382 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5383 bool exit_if_true = 0;
5384 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5385 uint32_t exit_point;
5386
5387 if (ra) {
5388 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5389 }
5390 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5391 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5392 if (!exit_addr) {
5393 goto jit_failure;
5394 }
5395 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5396 } else {
5397 smart_branch_opcode = 0;
5398 exit_addr = NULL;
5399 }
5400 if (!zend_jit_type_check(&dasm_state, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5401 goto jit_failure;
5402 }
5403 goto done;
5404 case ZEND_RETURN:
5405 op1_info = OP1_INFO();
5406 CHECK_OP1_TRACE_TYPE();
5407 if (opline->op1_type == IS_CONST) {
5408 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5409 } else if (op1_type != IS_UNKNOWN) {
5410 res_type = op1_type;
5411 }
5412 if (op_array->type == ZEND_EVAL_CODE
5413 // TODO: support for top-level code
5414 || !op_array->function_name
5415 // TODO: support for IS_UNDEF ???
5416 || (op1_info & MAY_BE_UNDEF)) {
5417 if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5418 goto jit_failure;
5419 }
5420 } else {
5421 int j;
5422 int may_throw = 0;
5423 bool left_frame = 0;
5424
5425 if (!zend_jit_return(&dasm_state, opline, op_array,
5426 op1_info, OP1_REG_ADDR())) {
5427 goto jit_failure;
5428 }
5429 if (op_array->last_var > 100) {
5430 /* To many CVs to unroll */
5431 if (!zend_jit_free_cvs(&dasm_state)) {
5432 goto jit_failure;
5433 }
5434 left_frame = 1;
5435 }
5436 if (!left_frame) {
5437 for (j = 0 ; j < op_array->last_var; j++) {
5438 uint32_t info;
5439 uint8_t type;
5440
5441 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5442 type = STACK_TYPE(stack, j);
5443 info = zend_jit_trace_type_to_info_ex(type, info);
5444 if (opline->op1_type == IS_CV
5445 && EX_VAR_TO_NUM(opline->op1.var) == j
5446 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5447 if (JIT_G(current_frame)
5448 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5449 continue;
5450 } else {
5451 info |= MAY_BE_NULL;
5452 }
5453 }
5454 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5455 if (!left_frame) {
5456 left_frame = 1;
5457 if (!zend_jit_leave_frame(&dasm_state)) {
5458 goto jit_failure;
5459 }
5460 }
5461 if (!zend_jit_free_cv(&dasm_state, info, j)) {
5462 goto jit_failure;
5463 }
5464 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5465 if (info & MAY_BE_RC1) {
5466 may_throw = 1;
5467 }
5468 }
5469 }
5470 }
5471 }
5472 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
5473 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5474 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5475 goto jit_failure;
5476 }
5477 }
5478 goto done;
5479 case ZEND_BOOL:
5480 case ZEND_BOOL_NOT:
5481 op1_info = OP1_INFO();
5482 CHECK_OP1_TRACE_TYPE();
5483 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5484 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5485 -1, -1,
5486 zend_may_throw(opline, ssa_op, op_array, ssa),
5487 opline->opcode, NULL)) {
5488 goto jit_failure;
5489 }
5490 goto done;
5491 case ZEND_JMPZ:
5492 case ZEND_JMPNZ:
5493 case ZEND_JMPZ_EX:
5494 case ZEND_JMPNZ_EX:
5495 op1_info = OP1_INFO();
5496 CHECK_OP1_TRACE_TYPE();
5497 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5498 const zend_op *exit_opline = NULL;
5499 uint32_t exit_point;
5500
5501 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5502 /* taken branch */
5503 if (opline->opcode == ZEND_JMPNZ_EX) {
5504 smart_branch_opcode = ZEND_JMPZ_EX;
5505 } else if (opline->opcode == ZEND_JMPZ_EX) {
5506 smart_branch_opcode = ZEND_JMPNZ_EX;
5507 } else if (opline->opcode == ZEND_JMPNZ) {
5508 smart_branch_opcode = ZEND_JMPZ;
5509 } else {
5510 smart_branch_opcode = ZEND_JMPNZ;
5511 }
5512 exit_opline = opline + 1;
5513 } else if ((p+1)->opline == opline + 1) {
5514 /* not taken branch */
5515 smart_branch_opcode = opline->opcode;
5516 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5517 } else {
5518 ZEND_UNREACHABLE();
5519 }
5520 if (ra) {
5521 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5522 }
5523 if (!(op1_info & MAY_BE_GUARD)
5524 && has_concrete_type(op1_info)
5525 && concrete_type(op1_info) <= IS_TRUE) {
5526 /* unconditional branch */
5527 exit_addr = NULL;
5528 } else if (opline->result_type == IS_TMP_VAR) {
5529 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5530 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5531
5532 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5533 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5534 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5535 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5536 if (!exit_addr) {
5537 goto jit_failure;
5538 }
5539 } else {
5540 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5541 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5542 if (!exit_addr) {
5543 goto jit_failure;
5544 }
5545 }
5546 } else {
5547 ZEND_UNREACHABLE();
5548 }
5549 if (opline->result_type == IS_UNDEF) {
5550 res_addr = 0;
5551 } else {
5552 res_addr = RES_REG_ADDR();
5553 }
5554 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
5555 op1_info, OP1_REG_ADDR(), res_addr,
5556 -1, -1,
5557 zend_may_throw(opline, ssa_op, op_array, ssa),
5558 smart_branch_opcode, exit_addr)) {
5559 goto jit_failure;
5560 }
5561 goto done;
5562 case ZEND_ISSET_ISEMPTY_CV:
5563 if ((opline->extended_value & ZEND_ISEMPTY)) {
5564 // TODO: support for empty() ???
5565 break;
5566 }
5567 op1_info = OP1_INFO();
5568 op1_addr = OP1_REG_ADDR();
5569 if (orig_op1_type != IS_UNKNOWN
5570 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5571 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5572 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5573 goto jit_failure;
5574 }
5575 if (opline->op1_type == IS_CV
5576 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5577 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5578 }
5579 } else {
5580 CHECK_OP1_TRACE_TYPE();
5581 }
5582 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5583 bool exit_if_true = 0;
5584 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5585 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5586
5587 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5588 if (!exit_addr) {
5589 goto jit_failure;
5590 }
5591 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5592 } else {
5593 smart_branch_opcode = 0;
5594 exit_addr = NULL;
5595 }
5596 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
5597 op1_info, op1_addr,
5598 smart_branch_opcode, -1, -1, exit_addr)) {
5599 goto jit_failure;
5600 }
5601 goto done;
5602 case ZEND_IN_ARRAY:
5603 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5604 break;
5605 }
5606 op1_info = OP1_INFO();
5607 op1_addr = OP1_REG_ADDR();
5608 CHECK_OP1_TRACE_TYPE();
5609 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5610 break;
5611 }
5612 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5613 bool exit_if_true = 0;
5614 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5615 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5616
5617 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5618 if (!exit_addr) {
5619 goto jit_failure;
5620 }
5621 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5622 } else {
5623 smart_branch_opcode = 0;
5624 exit_addr = NULL;
5625 }
5626 if (!zend_jit_in_array(&dasm_state, opline,
5627 op1_info, op1_addr,
5628 smart_branch_opcode, -1, -1, exit_addr)) {
5629 goto jit_failure;
5630 }
5631 goto done;
5632 case ZEND_FETCH_DIM_FUNC_ARG:
5633 if (!JIT_G(current_frame)
5634 || !JIT_G(current_frame)->call
5635 || !JIT_G(current_frame)->call->func
5636 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5637 break;
5638 }
5639 ZEND_FALLTHROUGH;
5640 case ZEND_FETCH_DIM_R:
5641 case ZEND_FETCH_DIM_IS:
5642 case ZEND_FETCH_LIST_R:
5643 op1_info = OP1_INFO();
5644 op1_addr = OP1_REG_ADDR();
5645 if (orig_op1_type != IS_UNKNOWN
5646 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5647 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5648 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5649 goto jit_failure;
5650 }
5651 if (opline->op1_type == IS_CV
5652 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5653 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5654 if (ssa_op->op1_def >= 0) {
5655 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5656 }
5657 }
5658 } else {
5659 CHECK_OP1_TRACE_TYPE();
5660 }
5661 op2_info = OP2_INFO();
5662 CHECK_OP2_TRACE_TYPE();
5663 res_info = RES_INFO();
5664 avoid_refcounting =
5665 ssa_op->op1_use >= 0 &&
5666 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5667 if (op1_info & MAY_BE_PACKED_GUARD) {
5668 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5669 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5670 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5671 && MAY_BE_PACKED(op1_info)
5672 && MAY_BE_HASH(op1_info)
5673 && orig_op1_type != IS_UNKNOWN) {
5674 op1_info |= MAY_BE_PACKED_GUARD;
5675 if (orig_op1_type & IS_TRACE_PACKED) {
5676 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5677 if (op1_type != IS_UNKNOWN) {
5678 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5679 }
5680 } else {
5681 op1_info &= ~MAY_BE_ARRAY_PACKED;
5682 if (op1_type != IS_UNKNOWN) {
5683 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5684 }
5685 }
5686 }
5687 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
5688 op1_info, op1_addr, avoid_refcounting,
5689 op2_info, res_info, RES_REG_ADDR(), val_type)) {
5690 goto jit_failure;
5691 }
5692 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5693 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5694 }
5695 goto done;
5696 case ZEND_FETCH_DIM_W:
5697 case ZEND_FETCH_DIM_RW:
5698 // case ZEND_FETCH_DIM_UNSET:
5699 case ZEND_FETCH_LIST_W:
5700 if (opline->op1_type != IS_CV
5701 && (orig_op1_type == IS_UNKNOWN
5702 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5703 break;
5704 }
5705 op1_info = OP1_INFO();
5706 op1_addr = OP1_REG_ADDR();
5707 if (opline->op1_type == IS_VAR) {
5708 if (orig_op1_type != IS_UNKNOWN
5709 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5710 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5711 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5712 goto jit_failure;
5713 }
5714 } else {
5715 break;
5716 }
5717 }
5718 if (orig_op1_type != IS_UNKNOWN
5719 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5720 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5721 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5722 goto jit_failure;
5723 }
5724 if (opline->op1_type == IS_CV
5725 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5726 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5727 }
5728 } else {
5729 CHECK_OP1_TRACE_TYPE();
5730 }
5731 op2_info = OP2_INFO();
5732 CHECK_OP2_TRACE_TYPE();
5733 op1_def_info = OP1_DEF_INFO();
5734 if (!zend_jit_fetch_dim(&dasm_state, opline,
5735 op1_info, op1_addr, op2_info, RES_REG_ADDR(), val_type)) {
5736 goto jit_failure;
5737 }
5738 if (ssa_op->result_def >= 0
5739 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5740 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5741 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5742 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5743 }
5744 goto done;
5745 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5746 if ((opline->extended_value & ZEND_ISEMPTY)) {
5747 // TODO: support for empty() ???
5748 break;
5749 }
5750 op1_info = OP1_INFO();
5751 op1_addr = OP1_REG_ADDR();
5752 if (orig_op1_type != IS_UNKNOWN
5753 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5754 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5755 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5756 goto jit_failure;
5757 }
5758 if (opline->op1_type == IS_CV
5759 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5760 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5761 }
5762 } else {
5763 CHECK_OP1_TRACE_TYPE();
5764 }
5765 op2_info = OP2_INFO();
5766 CHECK_OP2_TRACE_TYPE();
5767 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5768 bool exit_if_true = 0;
5769 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5770 uint32_t exit_point;
5771
5772 if (ra) {
5773 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
5774 }
5775 if (ssa_op->op1_use >= 0
5776 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5777 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5778 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5779 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5780
5781 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5782 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5783 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5784 } else {
5785 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5786 }
5787 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5788 if (!exit_addr) {
5789 goto jit_failure;
5790 }
5791 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5792 } else {
5793 smart_branch_opcode = 0;
5794 exit_addr = NULL;
5795 }
5796 avoid_refcounting =
5797 ssa_op->op1_use >= 0 &&
5798 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5799 if (op1_info & MAY_BE_PACKED_GUARD) {
5800 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5801 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5802 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5803 && MAY_BE_PACKED(op1_info)
5804 && MAY_BE_HASH(op1_info)
5805 && orig_op1_type != IS_UNKNOWN) {
5806 op1_info |= MAY_BE_PACKED_GUARD;
5807 if (orig_op1_type & IS_TRACE_PACKED) {
5808 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5809 } else {
5810 op1_info &= ~MAY_BE_ARRAY_PACKED;
5811 }
5812 }
5813 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
5814 op1_info, op1_addr, avoid_refcounting,
5815 op2_info, val_type,
5816 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5817 smart_branch_opcode, -1, -1,
5818 exit_addr)) {
5819 goto jit_failure;
5820 }
5821 goto done;
5822 case ZEND_FETCH_OBJ_FUNC_ARG:
5823 if (!JIT_G(current_frame)
5824 || !JIT_G(current_frame)->call
5825 || !JIT_G(current_frame)->call->func
5826 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5827 break;
5828 }
5829 ZEND_FALLTHROUGH;
5830 case ZEND_FETCH_OBJ_R:
5831 case ZEND_FETCH_OBJ_IS:
5832 case ZEND_FETCH_OBJ_W:
5833 on_this = delayed_fetch_this = 0;
5834 avoid_refcounting = 0;
5835 if (opline->op2_type != IS_CONST
5836 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5837 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5838 break;
5839 }
5840 ce = NULL;
5841 ce_is_instanceof = 0;
5842 op1_indirect = 0;
5843 if (opline->op1_type == IS_UNUSED) {
5844 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
5845 ce = op_array->scope;
5846 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
5847 op1_addr = 0;
5848 on_this = 1;
5849 } else {
5850 op1_info = OP1_INFO();
5851 if (!(op1_info & MAY_BE_OBJECT)) {
5852 break;
5853 }
5854 op1_addr = OP1_REG_ADDR();
5855 if (opline->op1_type == IS_VAR
5856 && opline->opcode == ZEND_FETCH_OBJ_W) {
5857 if (orig_op1_type != IS_UNKNOWN
5858 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5859 op1_indirect = 1;
5860 if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type,
5861 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5862 goto jit_failure;
5863 }
5864 }
5865 }
5866 if (orig_op1_type != IS_UNKNOWN
5867 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5868 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5869 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5870 goto jit_failure;
5871 }
5872 if (opline->op1_type == IS_CV
5873 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5874 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
5875 }
5876 } else {
5877 CHECK_OP1_TRACE_TYPE();
5878 }
5879 if (!(op1_info & MAY_BE_OBJECT)) {
5880 break;
5881 }
5882 if (ssa->var_info && ssa->ops) {
5883 if (ssa_op->op1_use >= 0) {
5884 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5885 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5886 ce = op1_ssa->ce;
5887 ce_is_instanceof = op1_ssa->is_instanceof;
5888 }
5889 }
5890 }
5891 if (ssa_op->op1_use >= 0) {
5892 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
5893 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5894 }
5895 if (delayed_fetch_this) {
5896 on_this = 1;
5897 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
5898 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
5899 } else if (op_array_ssa->ops
5900 && op_array_ssa->vars
5901 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
5902 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
5903 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
5904 }
5905 }
5906 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
5907 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
5908 on_this, delayed_fetch_this, avoid_refcounting, op1_ce, val_type,
5909 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
5910 goto jit_failure;
5911 }
5912 goto done;
5913 case ZEND_BIND_GLOBAL:
5914 orig_opline = opline;
5915 orig_ssa_op = ssa_op;
5916 while (1) {
5917 if (!ssa->ops || !ssa->var_info) {
5918 op1_info = MAY_BE_ANY|MAY_BE_REF;
5919 } else {
5920 op1_info = OP1_INFO();
5921 }
5922 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5923 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5924 }
5925 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
5926 goto jit_failure;
5927 }
5928 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
5929 opline++;
5930 ssa_op++;
5931 } else {
5932 break;
5933 }
5934 }
5935 opline = orig_opline;
5936 ssa_op = orig_ssa_op;
5937 goto done;
5938 case ZEND_RECV:
5939 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
5940 goto jit_failure;
5941 }
5942 goto done;
5943 case ZEND_RECV_INIT:
5944 orig_opline = opline;
5945 orig_ssa_op = ssa_op;
5946 while (1) {
5947 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
5948 (opline + 1)->opcode != ZEND_RECV_INIT,
5949 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5950 goto jit_failure;
5951 }
5952 if ((opline+1)->opcode == ZEND_RECV_INIT) {
5953 opline++;
5954 ssa_op++;
5955 } else {
5956 break;
5957 }
5958 }
5959 opline = orig_opline;
5960 ssa_op = orig_ssa_op;
5961 goto done;
5962 case ZEND_FREE:
5963 case ZEND_FE_FREE:
5964 op1_info = OP1_INFO();
5965 if (!zend_jit_free(&dasm_state, opline, op1_info,
5966 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5967 goto jit_failure;
5968 }
5969 goto done;
5970 case ZEND_ECHO:
5971 op1_info = OP1_INFO();
5972 CHECK_OP1_TRACE_TYPE();
5973 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5974 break;
5975 }
5976 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
5977 goto jit_failure;
5978 }
5979 goto done;
5980 case ZEND_STRLEN:
5981 op1_info = OP1_INFO();
5982 op1_addr = OP1_REG_ADDR();
5983 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
5984 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
5985 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5986 goto jit_failure;
5987 }
5988 if (opline->op1_type == IS_CV
5989 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5990 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5991 }
5992 } else {
5993 CHECK_OP1_TRACE_TYPE();
5994 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
5995 break;
5996 }
5997 }
5998 if (!zend_jit_strlen(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR())) {
5999 goto jit_failure;
6000 }
6001 goto done;
6002 case ZEND_COUNT:
6003 op1_info = OP1_INFO();
6004 op1_addr = OP1_REG_ADDR();
6005 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
6006 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6007 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6008 goto jit_failure;
6009 }
6010 if (opline->op1_type == IS_CV
6011 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6012 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6013 }
6014 } else {
6015 CHECK_OP1_TRACE_TYPE();
6016 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6017 break;
6018 }
6019 }
6020 if (!zend_jit_count(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6021 goto jit_failure;
6022 }
6023 goto done;
6024 case ZEND_FETCH_THIS:
6025 delayed_fetch_this = 0;
6026 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6027 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6028 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6029 delayed_fetch_this = 1;
6030 }
6031 }
6032 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) {
6033 goto jit_failure;
6034 }
6035 goto done;
6036 case ZEND_SWITCH_LONG:
6037 case ZEND_SWITCH_STRING:
6038 case ZEND_MATCH:
6039 if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6040 goto jit_failure;
6041 }
6042 goto done;
6043 case ZEND_VERIFY_RETURN_TYPE:
6044 if (opline->op1_type == IS_UNUSED) {
6045 /* Always throws */
6046 break;
6047 }
6048 if (opline->op1_type == IS_CONST) {
6049 /* TODO Different instruction format, has return value */
6050 break;
6051 }
6052 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6053 /* Not worth bothering with */
6054 break;
6055 }
6056 op1_info = OP1_INFO();
6057 CHECK_OP1_TRACE_TYPE();
6058 if (op1_info & MAY_BE_REF) {
6059 /* TODO May need reference unwrapping. */
6060 break;
6061 }
6062 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, op1_info)) {
6063 goto jit_failure;
6064 }
6065 goto done;
6066 case ZEND_FE_RESET_R:
6067 op1_info = OP1_INFO();
6068 CHECK_OP1_TRACE_TYPE();
6069 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6070 break;
6071 }
6072 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
6073 goto jit_failure;
6074 }
6075 goto done;
6076 case ZEND_FE_FETCH_R:
6077 op1_info = OP1_INFO();
6078 CHECK_OP1_TRACE_TYPE();
6079 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6080 break;
6081 }
6082 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6083 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6084 uint32_t exit_point;
6085
6086 if ((p+1)->opline == exit_opline) {
6087 /* taken branch (exit from loop) */
6088 exit_opline = opline;
6089 smart_branch_opcode = ZEND_NOP;
6090 } else if ((p+1)->opline == opline + 1) {
6091 /* not taken branch (loop) */
6092 smart_branch_opcode = ZEND_JMP;
6093 } else {
6094 ZEND_UNREACHABLE();
6095 }
6096 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6097 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6098 if (!exit_addr) {
6099 goto jit_failure;
6100 }
6101 } else {
6102 ZEND_UNREACHABLE();
6103 }
6104 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
6105 -1, smart_branch_opcode, exit_addr)) {
6106 goto jit_failure;
6107 }
6108 goto done;
6109 case ZEND_FETCH_CONSTANT:
6110 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6111 goto jit_failure;
6112 }
6113 goto done;
6114 case ZEND_INIT_METHOD_CALL:
6115 if (opline->op2_type != IS_CONST
6116 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6117 break;
6118 }
6119 on_this = delayed_fetch_this = 0;
6120 ce = NULL;
6121 ce_is_instanceof = 0;
6122 if (opline->op1_type == IS_UNUSED) {
6123 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6124 ce = op_array->scope;
6125 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
6126 op1_addr = 0;
6127 on_this = 1;
6128 } else {
6129 op1_info = OP1_INFO();
6130 op1_addr = OP1_REG_ADDR();
6131 if (polymorphic_side_trace) {
6132 op1_info = MAY_BE_OBJECT;
6133 op1_addr = 0;
6134 } else if (orig_op1_type != IS_UNKNOWN
6135 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6136 if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
6137 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6138 goto jit_failure;
6139 }
6140 if (opline->op1_type == IS_CV
6141 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6142 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6143 }
6144 } else {
6145 CHECK_OP1_TRACE_TYPE();
6146 }
6147 if (ssa->var_info && ssa->ops) {
6148 if (ssa_op->op1_use >= 0) {
6149 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6150 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6151 ce = op1_ssa->ce;
6152 ce_is_instanceof = op1_ssa->is_instanceof;
6153 }
6154 }
6155 }
6156 if (ssa_op->op1_use >= 0) {
6157 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6158 }
6159 if (delayed_fetch_this) {
6160 on_this = 1;
6161 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6162 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6163 } else if (op_array_ssa->ops
6164 && op_array_ssa->vars
6165 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6166 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6167 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6168 }
6169 }
6170 frame_flags = TRACE_FRAME_MASK_NESTED;
6171 if (!zend_jit_init_method_call(&dasm_state, opline,
6172 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6173 op_array, ssa, ssa_op, frame->call_level,
6174 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6175 p + 1, peek_checked_stack - checked_stack, polymorphic_side_trace)) {
6176 goto jit_failure;
6177 }
6178 goto done;
6179 case ZEND_INIT_DYNAMIC_CALL:
6180 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6181 break;
6182 }
6183 op2_info = OP2_INFO();
6184 CHECK_OP2_TRACE_TYPE();
6185 frame_flags = TRACE_FRAME_MASK_NESTED;
6186 if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6187 goto jit_failure;
6188 }
6189 goto done;
6190 case ZEND_SEND_ARRAY:
6191 case ZEND_SEND_UNPACK:
6192 if (JIT_G(current_frame)
6193 && JIT_G(current_frame)->call) {
6194 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6195 }
6196 break;
6197 case ZEND_ROPE_INIT:
6198 case ZEND_ROPE_ADD:
6199 case ZEND_ROPE_END:
6200 op2_info = OP2_INFO();
6201 CHECK_OP2_TRACE_TYPE();
6202 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6203 break;
6204 }
6205 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
6206 goto jit_failure;
6207 }
6208 goto done;
6209 default:
6210 break;
6211 }
6212 }
6213
6214 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6215 gen_handler = 1;
6216 op1_info = OP1_INFO();
6217 op2_info = OP2_INFO();
6218 if (op1_info & MAY_BE_GUARD) {
6219 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6220 }
6221 if (op2_info & MAY_BE_GUARD) {
6222 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6223 }
6224 if (!zend_jit_trace_handler(&dasm_state, op_array, opline,
6225 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6226 goto jit_failure;
6227 }
6228 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
6229 if (opline->opcode == ZEND_NEW && opline->result_type != IS_UNUSED) {
6230 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6231 }
6232 if (zend_jit_may_be_polymorphic_call(opline) ||
6233 zend_jit_may_be_modified((p+1)->func, op_array)) {
6234 if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) {
6235 goto jit_failure;
6236 }
6237 }
6238 }
6239 }
6240
6241 done:
6242 polymorphic_side_trace = 0;
6243 switch (opline->opcode) {
6244 case ZEND_DO_FCALL:
6245 case ZEND_DO_ICALL:
6246 case ZEND_DO_UCALL:
6247 case ZEND_DO_FCALL_BY_NAME:
6248 case ZEND_CALLABLE_CONVERT:
6249 frame->call_level--;
6250 }
6251
6252 if (ra) {
6253 zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra);
6254 }
6255
6256 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6257 && STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var)) > ZREG_NUM) {
6258 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6259 }
6260
6261 if (opline->opcode == ZEND_ROPE_INIT) {
6262 /* clear stack slots used by rope */
6263 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6264 uint32_t count =
6265 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6266
6267 do {
6268 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6269 var++;
6270 count--;
6271 } while (count);
6272 }
6273
6274 if (ssa_op) {
6275 zend_ssa_range tmp;
6276
6277 /* Keep information about known types on abstract stack */
6278 if (ssa_op->result_def >= 0) {
6279 uint8_t type = IS_UNKNOWN;
6280
6281 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6282 || send_result) {
6283 /* we didn't set result variable */
6284 type = IS_UNKNOWN;
6285 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6286 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6287 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6288 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6289 if (opline->op1_type != IS_CONST) {
6290 /* copy */
6291 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6292 }
6293 } else if (opline->opcode == ZEND_ASSIGN) {
6294 if (opline->op2_type != IS_CONST
6295 && ssa_op->op1_use >= 0
6296 /* assignment to typed reference may cause conversion */
6297 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6298 /* copy */
6299 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6300 }
6301 } else if (opline->opcode == ZEND_POST_INC
6302 || opline->opcode == ZEND_POST_DEC) {
6303 /* copy */
6304 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6305 }
6306 if (opline->opcode == ZEND_JMP_SET
6307 || opline->opcode == ZEND_COALESCE
6308 || opline->opcode == ZEND_JMP_NULL) {
6309 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6310 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6311 } else if ((p+1)->opline != (opline + 1)) {
6312 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6313 }
6314 } else {
6315 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6316 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->result_def]));
6317 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6318 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6319 }
6320 if (type != IS_UNKNOWN) {
6321 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6322 if (opline->opcode == ZEND_FETCH_THIS
6323 && delayed_fetch_this) {
6324 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_THIS);
6325 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6326 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_TRY_ADDREF);
6327 } else if (ra && ra[ssa_op->result_def]) {
6328 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6329 ra[ssa_op->result_def]->flags & ZREG_STORE);
6330 }
6331 }
6332 }
6333
6334 if (type == IS_LONG
6335 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) {
6336 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6337 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6338 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6339 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6340 ssa->var_info[ssa_op->result_def].has_range = 1;
6341 }
6342 }
6343 if (ssa_op->op1_def >= 0
6344 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6345 || opline->result_type != IS_CV
6346 || opline->result.var != opline->op1.var)) {
6347 uint8_t type = IS_UNKNOWN;
6348
6349 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6350 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6351 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6352 } else if (opline->opcode == ZEND_ASSIGN) {
6353 if (!(OP1_INFO() & MAY_BE_REF)
6354 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6355 if (opline->op2_type != IS_CONST) {
6356 /* copy */
6357 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6358 }
6359 }
6360 } else if (opline->opcode == ZEND_SEND_VAR
6361 || opline->opcode == ZEND_CAST
6362 || opline->opcode == ZEND_QM_ASSIGN
6363 || opline->opcode == ZEND_JMP_SET
6364 || opline->opcode == ZEND_COALESCE
6365 || opline->opcode == ZEND_JMP_NULL
6366 || opline->opcode == ZEND_FE_RESET_R) {
6367 /* keep old value */
6368 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6369 }
6370 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6371 (gen_handler || type == IS_UNKNOWN || !ra ||
6372 (!ra[ssa_op->op1_def] &&
6373 !(ssa->vars[ssa_op->op1_def].no_val &&
6374 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6375 (opline->opcode == ZEND_QM_ASSIGN ||
6376 opline->opcode == ZEND_SEND_VAR ||
6377 opline->opcode == ZEND_SEND_VAR_EX ||
6378 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6379 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6380 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6381 if (type != IS_UNKNOWN) {
6382 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6383 if (ra && ra[ssa_op->op1_def]) {
6384 uint8_t flags = ra[ssa_op->op1_def]->flags & ZREG_STORE;
6385
6386 if (ssa_op->op1_use >= 0) {
6387 if (opline->opcode == ZEND_SEND_VAR
6388 || opline->opcode == ZEND_CAST
6389 || opline->opcode == ZEND_QM_ASSIGN
6390 || opline->opcode == ZEND_JMP_SET
6391 || opline->opcode == ZEND_COALESCE
6392 || opline->opcode == ZEND_JMP_NULL
6393 || opline->opcode == ZEND_FE_RESET_R) {
6394 if (!ra[ssa_op->op1_use]) {
6395 flags |= ZREG_LOAD;
6396 }
6397 }
6398 }
6399 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, flags);
6400 }
6401 }
6402 if (type == IS_LONG
6403 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6404 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6405 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6406 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6407 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6408 ssa->var_info[ssa_op->op1_def].has_range = 1;
6409 }
6410 }
6411 if (ssa_op->op2_def >= 0
6412 && (opline->opcode != ZEND_ASSIGN
6413 || opline->op1_type != IS_CV
6414 || opline->op1.var != opline->op2.var)) {
6415 uint8_t type = IS_UNKNOWN;
6416
6417 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6418 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6419 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6420 } else if (opline->opcode == ZEND_ASSIGN) {
6421 /* keep old value */
6422 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6423 }
6424 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6425 (gen_handler || type == IS_UNKNOWN || !ra ||
6426 (!ra[ssa_op->op2_def] &&
6427 !(ssa->vars[ssa_op->op2_def].no_val &&
6428 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6429 opline->opcode == ZEND_ASSIGN))));
6430 if (type != IS_UNKNOWN) {
6431 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6432 if (ra && ra[ssa_op->op2_def]) {
6433 uint8_t flags = ra[ssa_op->op2_def]->flags & ZREG_STORE;
6434
6435 if (ssa_op->op2_use >= 0) {
6436 if (opline->opcode == ZEND_ASSIGN) {
6437 if (!ra[ssa_op->op2_use]
6438 || ra[ssa_op->op2_use]->reg != ra[ssa_op->op2_def]->reg) {
6439 flags |= ZREG_LOAD;
6440 }
6441 }
6442 }
6443 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def]->reg, flags);
6444 }
6445 }
6446 if (type == IS_LONG
6447 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) {
6448 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6449 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6450 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6451 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6452 ssa->var_info[ssa_op->op2_def].has_range = 1;
6453 }
6454 }
6455
6456 switch (opline->opcode) {
6457 case ZEND_ASSIGN_DIM:
6458 case ZEND_ASSIGN_OBJ:
6459 case ZEND_ASSIGN_STATIC_PROP:
6460 case ZEND_ASSIGN_DIM_OP:
6461 case ZEND_ASSIGN_OBJ_OP:
6462 case ZEND_ASSIGN_STATIC_PROP_OP:
6463 case ZEND_ASSIGN_OBJ_REF:
6464 case ZEND_ASSIGN_STATIC_PROP_REF:
6465 /* OP_DATA */
6466 ssa_op++;
6467 opline++;
6468 if (ssa_op->op1_def >= 0) {
6469 uint8_t type = IS_UNKNOWN;
6470
6471 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6472 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6473 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6474 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6475 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6476 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6477 /* keep old value */
6478 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6479 }
6480 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6481 (gen_handler || type == IS_UNKNOWN || !ra || !ra[ssa_op->op1_def]));
6482 if (type != IS_UNKNOWN) {
6483 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6484 if (ra && ra[ssa_op->op1_def]) {
6485 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6486 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6487 }
6488 }
6489 if (type == IS_LONG
6490 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6491 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6492 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6493 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6494 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6495 ssa->var_info[ssa_op->op1_def].has_range = 1;
6496 }
6497 }
6498 ssa_op++;
6499 break;
6500 case ZEND_RECV_INIT:
6501 ssa_op++;
6502 opline++;
6503 while (opline->opcode == ZEND_RECV_INIT) {
6504 if (ssa_op->result_def >= 0) {
6505 uint8_t type = IS_UNKNOWN;
6506
6507 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6508 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6509 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6510 }
6511 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6512 (gen_handler || !ra || !ra[ssa_op->result_def]));
6513 if (ra && ra[ssa_op->result_def]) {
6514 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg,
6515 ra[ssa_op->result_def]->flags & ZREG_STORE);
6516 }
6517 }
6518 ssa_op++;
6519 opline++;
6520 }
6521 break;
6522 case ZEND_BIND_GLOBAL:
6523 ssa_op++;
6524 opline++;
6525 while (opline->opcode == ZEND_BIND_GLOBAL) {
6526 if (ssa_op->op1_def >= 0) {
6527 uint8_t type = IS_UNKNOWN;
6528
6529 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6530 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6531 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6532 }
6533 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6534 (gen_handler || !ra || !ra[ssa_op->op1_def]));
6535 if (ra && ra[ssa_op->op1_def]) {
6536 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg,
6537 ra[ssa_op->op1_def]->flags & ZREG_STORE);
6538 }
6539 }
6540 ssa_op++;
6541 opline++;
6542 }
6543 break;
6544 default:
6545 ssa_op += zend_jit_trace_op_len(opline);
6546 break;
6547 }
6548
6549 if (send_result) {
6550 ssa_op++;
6551 p++;
6552 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6553 p++;
6554 }
6555 send_result = 0;
6556 }
6557 }
6558 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6559 call = frame->call;
6560 assert(call && &call->func->op_array == p->op_array);
6561
6562 if (opline->opcode == ZEND_DO_UCALL
6563 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6564 || opline->opcode == ZEND_DO_FCALL) {
6565
6566 frame->call_opline = opline;
6567
6568 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6569 if (opline > op_array->opcodes) {
6570 const zend_op *prev_opline = opline - 1;
6571
6572 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6573 prev_opline--;
6574 }
6575 JIT_G(current_frame) = call;
6576 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6577 || prev_opline->opcode == ZEND_SEND_UNPACK
6578 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6579 && p->op_array->num_args
6580 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6581 && ((p+1)->op == ZEND_JIT_TRACE_VM
6582 || (p+1)->op == ZEND_JIT_TRACE_END)
6583 && (TRACE_FRAME_NUM_ARGS(call) < 0
6584 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6585 && !zend_jit_trace_opline_guard(&dasm_state, (p+1)->opline)) {
6586 goto jit_failure;
6587 }
6588 JIT_G(current_frame) = frame;
6589 }
6590 }
6591
6592 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6593 p++;
6594 break;
6595 }
6596 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6597 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6598 TRACE_FRAME_SET_THIS_CHECKED(call);
6599 }
6600 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6601 TRACE_FRAME_SET_THIS_CHECKED(call);
6602 }
6603 op_array = (zend_op_array*)p->op_array;
6604 jit_extension =
6605 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6606 op_array_ssa = &jit_extension->func_info.ssa;
6607 frame->call = call->prev;
6608 call->prev = frame;
6609 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6610 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6611 } else {
6612 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6613 }
6614 JIT_G(current_frame) = frame = call;
6615 stack = frame->stack;
6616 if (ra) {
6617 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6618
6619 for (i = 0; i < op_array->last_var; i++,j++) {
6620 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6621 if ((ssa->var_info[j].type & MAY_BE_GUARD) != 0) {
6622 uint8_t op_type;
6623
6624 ssa->var_info[j].type &= ~MAY_BE_GUARD;
6625 op_type = concrete_type(ssa->var_info[j].type);
6626 if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) {
6627 goto jit_failure;
6628 }
6629 SET_STACK_TYPE(stack, i, op_type, 1);
6630 }
6631 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6632 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6633 goto jit_failure;
6634 }
6635 }
6636 }
6637 }
6638 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6639 op_array = (zend_op_array*)p->op_array;
6640 jit_extension =
6641 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6642 op_array_ssa = &jit_extension->func_info.ssa;
6643 top = frame;
6644 if (frame->prev) {
6645 checked_stack = frame->old_checked_stack;
6646 peek_checked_stack = frame->old_peek_checked_stack;
6647 frame = frame->prev;
6648 stack = frame->stack;
6649 ZEND_ASSERT(&frame->func->op_array == op_array);
6650 } else {
6651 frame = zend_jit_trace_ret_frame(frame, op_array);
6652 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6653 frame->used_stack = checked_stack = peek_checked_stack = 0;
6654 stack = frame->stack;
6655 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6656 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6657
6658 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6659 /* Initialize abstract stack using SSA */
6660 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6661 && has_concrete_type(ssa->var_info[j].type)) {
6662 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6663 } else {
6664 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6665 }
6666 }
6667 if (ra) {
6668 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6669 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6670 if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) {
6671 SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD);
6672 if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) {
6673 goto jit_failure;
6674 }
6675 }
6676 }
6677 }
6678 } else {
6679 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6680 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6681 }
6682 }
6683 opline = NULL;
6684 }
6685 JIT_G(current_frame) = frame;
6686 if (res_type != IS_UNKNOWN
6687 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6688 const zend_op *opline = (p+1)->opline - 1;
6689 if (opline->result_type != IS_UNUSED) {
6690 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6691 }
6692 }
6693 res_type = IS_UNKNOWN;
6694 } else if (p->op == ZEND_JIT_TRACE_END) {
6695 break;
6696 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6697 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6698 int num_args = -1;
6699
6700 if (init_opline
6701 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6702 num_args = init_opline->extended_value;
6703 }
6704
6705 call = top;
6706 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6707 call->prev = frame->call;
6708 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6709 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6710 if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
6711 TRACE_FRAME_SET_CLOSURE_CALL(call);
6712 }
6713 }
6714 if (init_opline) {
6715 if (init_opline->opcode != ZEND_NEW
6716 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6717 || init_opline->op1_type == IS_UNDEF
6718 || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6719 && ssa_op
6720 && (ssa_op-1)->op1_use >=0
6721 && ssa->var_info[(ssa_op-1)->op1_use].delayed_fetch_this))
6722 && (init_opline->opcode != ZEND_INIT_USER_CALL
6723 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6724 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6725 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6726 ) {
6727 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6728 } else if (init_opline->opcode == ZEND_NEW
6729 || (init_opline->opcode == ZEND_INIT_METHOD_CALL
6730 && init_opline->op1_type != IS_UNDEF
6731 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6732 && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) {
6733 TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call);
6734 }
6735 }
6736 frame->call = call;
6737 top = zend_jit_trace_call_frame(top, p->op_array);
6738 if (p->func) {
6739 if (p->func->type == ZEND_USER_FUNCTION) {
6740 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6741 zend_jit_op_array_trace_extension *jit_extension =
6742 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6743
6744 i = 0;
6745 while (i < p->op_array->num_args) {
6746 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6747 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6748 i++;
6749 }
6750 while (i < p->op_array->last_var) {
6751 if (jit_extension
6752 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6753 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6754 } else {
6755 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6756 }
6757 i++;
6758 }
6759 while (i < p->op_array->last_var + p->op_array->T) {
6760 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6761 i++;
6762 }
6763 } else {
6764 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
6765 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6766 }
6767 }
6768 } else {
6769 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
6770 for (i = 0; i < p->op_array->num_args; i++) {
6771 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6772 }
6773 }
6774 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6775 int skip_guard = 0;
6776
6777 if (init_opline) {
6778 zend_call_info *call_info = jit_extension->func_info.callee_info;
6779
6780 while (call_info) {
6781 if (call_info->caller_init_opline == init_opline
6782 && !call_info->is_prototype) {
6783 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
6784 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
6785 && init_opline->op1_type != IS_CONST) {
6786 break;
6787 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
6788 break;
6789 }
6790 }
6791 skip_guard = 1;
6792 break;
6793 }
6794 call_info = call_info->next_callee;
6795 }
6796 if (!skip_guard
6797 && !zend_jit_may_be_polymorphic_call(init_opline)
6798 && !zend_jit_may_be_modified(p->func, op_array)) {
6799 skip_guard = 1;
6800 }
6801 }
6802
6803 if (!skip_guard) {
6804 if (!opline) {
6805 zend_jit_trace_rec *q = p + 1;
6806 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
6807 q++;
6808 }
6809 opline = q->opline;
6810 ZEND_ASSERT(opline != NULL);
6811 }
6812 if (!zend_jit_init_fcall_guard(&dasm_state,
6813 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
6814 goto jit_failure;
6815 }
6816 }
6817 }
6818 }
6819 call->old_checked_stack = checked_stack;
6820 call->old_peek_checked_stack = peek_checked_stack;
6821 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6822 frame->call_level++;
6823 call->used_stack = checked_stack = peek_checked_stack = 0;
6824 } else {
6825 if (p->func) {
6826 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
6827 } else {
6828 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
6829 }
6830 switch (init_opline->opcode) {
6831 case ZEND_INIT_FCALL:
6832 case ZEND_INIT_FCALL_BY_NAME:
6833 case ZEND_INIT_NS_FCALL_BY_NAME:
6834 case ZEND_INIT_METHOD_CALL:
6835 case ZEND_INIT_DYNAMIC_CALL:
6836 //case ZEND_INIT_STATIC_METHOD_CALL:
6837 //case ZEND_INIT_USER_CALL:
6838 //case ZEND_NEW:
6839 checked_stack += call->used_stack;
6840 if (checked_stack > peek_checked_stack) {
6841 peek_checked_stack = checked_stack;
6842 }
6843 break;
6844 default:
6845 checked_stack = peek_checked_stack = 0;
6846 }
6847 }
6848 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
6849 call = frame->call;
6850 if (call) {
6851 checked_stack = call->old_checked_stack;
6852 peek_checked_stack = call->old_peek_checked_stack;
6853 top = call;
6854 frame->call = call->prev;
6855 }
6856 } else {
6857 ZEND_UNREACHABLE();
6858 }
6859 }
6860
6861 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
6862
6863 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
6864
6865 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6866 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6867 }
6868
6869 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
6870 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
6871 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6872 if (ra) {
6873 zend_ssa_phi *phi = ssa->blocks[1].phis;
6874
6875 while (phi) {
6876 if (ra[phi->ssa_var]
6877 && ra[phi->sources[1]]
6878 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
6879 && (ra[phi->ssa_var]->flags & (ZREG_LOAD|ZREG_STORE)) == 0
6880 && (ra[phi->sources[1]]->flags & (ZREG_LOAD|ZREG_STORE)) == 0) {
6881 /* Store actual type to memory to avoid deoptimization mistakes */
6882 /* TODO: Alternatively, we may try to update alredy generated deoptimization info */
6883 zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var));
6884 }
6885 phi = phi->next;
6886 }
6887 }
6888 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6889 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6890 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6891 goto jit_failure;
6892 }
6893 }
6894 t->link = ZEND_JIT_TRACE_NUM;
6895 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6896 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
6897 }
6898 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
6899 const void *timeout_exit_addr = NULL;
6900
6901 t->flags |= ZEND_JIT_TRACE_LOOP;
6902
6903 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6904 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6905 || (ra
6906 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
6907 /* Deoptimize to the first instruction of the loop */
6908 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
6909
6910 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6911 if (!timeout_exit_addr) {
6912 goto jit_failure;
6913 }
6914 } else {
6915 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6916 }
6917 }
6918
6919 zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
6920 }
6921 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
6922 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
6923 if (ra
6924 && (p-1)->op != ZEND_JIT_TRACE_ENTER
6925 && (p-1)->op != ZEND_JIT_TRACE_BACK
6926 && opline->opcode != ZEND_DO_UCALL
6927 && opline->opcode != ZEND_DO_FCALL
6928 && opline->opcode != ZEND_DO_FCALL_BY_NAME
6929 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
6930 if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL,
6931 stack, op_array->last_var + op_array->T, NULL, NULL, NULL, 0)) {
6932 goto jit_failure;
6933 }
6934 for (i = 0; i < op_array->last_var; i++) {
6935 int8_t reg = STACK_REG(stack, i);
6936 uint8_t type = STACK_TYPE(stack, i);
6937
6938 if (reg == ZREG_NONE
6939 && type != IS_UNKNOWN
6940 && type != STACK_MEM_TYPE(stack, i)) {
6941 if (!zend_jit_store_var_type(&dasm_state, i, type)) {
6942 return 0;
6943 }
6944 SET_STACK_TYPE(stack, i, type, 1);
6945 }
6946 }
6947 }
6948 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
6949 const void *timeout_exit_addr = NULL;
6950
6951 t->link = zend_jit_find_trace(p->opline->handler);
6952 if (t->link == 0) {
6953 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
6954 * by zend_jit_trace_exit() in another thread after this
6955 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
6956 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
6957 * the "_counter_handler" functions, and these are not registered
6958 * tracer functions */
6959 goto jit_failure;
6960 }
6961 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6962 && !zend_jit_set_ip(&dasm_state, p->opline)) {
6963 goto jit_failure;
6964 }
6965 if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
6966 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6967 }
6968 if (parent_trace
6969 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
6970 && zend_jit_traces[parent_trace].root == t->link) {
6971 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
6972 uint32_t exit_point;
6973
6974 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6975 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6976 }
6977 exit_point = zend_jit_trace_get_exit_point(zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM);
6978 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6979 if (!timeout_exit_addr) {
6980 goto jit_failure;
6981 }
6982 } else {
6983 timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
6984 }
6985 }
6986 zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
6987 } else {
6988 zend_jit_trace_return(&dasm_state, 0, NULL);
6989 }
6990 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
6991 zend_jit_trace_return(&dasm_state, 0, NULL);
6992 } else {
6993 // TODO: not implemented ???
6994 ZEND_ASSERT(0 && p->stop);
6995 }
6996
6997 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
6998 goto jit_failure;
6999 }
7000
7001 if (!zend_jit_trace_end(&dasm_state, t)) {
7002 goto jit_failure;
7003 }
7004
7005 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM,
7006 parent_trace ? SP_ADJ_JIT : ((zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET),
7007 parent_trace ? SP_ADJ_NONE : SP_ADJ_JIT);
7008
7009 if (handler) {
7010 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
7011 const zend_op_array *rec_op_array;
7012
7013 rec_op_array = op_array = trace_buffer->op_array;
7014 jit_extension =
7015 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7016 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7017 for (;;p++) {
7018 if (p->op == ZEND_JIT_TRACE_VM) {
7019 opline = p->opline;
7020 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7021 if (p->op_array == rec_op_array) {
7022 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7023 }
7024 op_array = p->op_array;
7025 jit_extension =
7026 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7027 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7028 op_array = p->op_array;
7029 jit_extension =
7030 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7031 } else if (p->op == ZEND_JIT_TRACE_END) {
7032 break;
7033 }
7034 }
7035 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7036 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7037 if (opline
7038 && (opline->opcode == ZEND_DO_UCALL
7039 || opline->opcode == ZEND_DO_FCALL
7040 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7041 || opline->opcode == ZEND_YIELD
7042 || opline->opcode == ZEND_YIELD_FROM
7043 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7044 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7045 }
7046 if (JIT_G(current_frame)
7047 && JIT_G(current_frame)->prev) {
7048 frame = JIT_G(current_frame)->prev;
7049 do {
7050 if (frame->call_opline) {
7051 op_array = &frame->func->op_array;
7052 jit_extension =
7053 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7054 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7055 }
7056 frame = frame->prev;
7057 } while (frame);
7058 }
7059 }
7060 }
7061
7062 jit_failure:
7063 dasm_free(&dasm_state);
7064
7065 if (name) {
7066 zend_string_release(name);
7067 }
7068
7069 jit_cleanup:
7070 /* Clean up used op_arrays */
7071 while (num_op_arrays > 0) {
7072 op_array = op_arrays[--num_op_arrays];
7073 jit_extension =
7074 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7075
7076 jit_extension->func_info.num = 0;
7077 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7078 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7079 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7080 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7081 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7082 }
7083
7084 zend_arena_release(&CG(arena), checkpoint);
7085
7086 JIT_G(current_frame) = NULL;
7087 JIT_G(current_trace) = NULL;
7088
7089 return handler;
7090 }
7091
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7092 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7093 {
7094 const void *handler = NULL;
7095 dasm_State* dasm_state = NULL;
7096 void *checkpoint;
7097 char name[32];
7098 const zend_op *opline;
7099 uint32_t stack_size;
7100 zend_jit_trace_stack *stack;
7101 bool original_handler = 0;
7102
7103 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7104 return dasm_labels[zend_lbtrace_escape];
7105 }
7106
7107 checkpoint = zend_arena_checkpoint(CG(arena));;
7108
7109 sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num);
7110
7111 dasm_init(&dasm_state, DASM_MAXSECTION);
7112 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
7113 dasm_setup(&dasm_state, dasm_actions);
7114
7115 zend_jit_align_func(&dasm_state);
7116
7117 /* Deoptimization */
7118 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7119 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
7120
7121 if (!zend_jit_trace_deoptimization(&dasm_state,
7122 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7123 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7124 stack, stack_size, NULL, NULL, NULL, 0)) {
7125 goto jit_failure;
7126 }
7127
7128 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7129 if (opline) {
7130 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7131 /* prevent endless loop */
7132 original_handler = 1;
7133 }
7134 zend_jit_set_ip_ex(&dasm_state, opline, original_handler);
7135 }
7136
7137 zend_jit_trace_return(&dasm_state, original_handler, opline);
7138
7139 handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM, SP_ADJ_JIT, SP_ADJ_NONE);
7140
7141 jit_failure:
7142 dasm_free(&dasm_state);
7143 zend_arena_release(&CG(arena), checkpoint);
7144 return handler;
7145 }
7146
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7147 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7148 {
7149 zend_jit_trace_stop ret;
7150 const void *handler;
7151 uint8_t orig_trigger;
7152 zend_jit_trace_info *t = NULL;
7153 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7154 bool do_bailout = 0;
7155
7156 zend_shared_alloc_lock();
7157
7158 /* Checks under lock */
7159 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7160 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7161 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7162 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7163 } else {
7164 zend_try {
7165 SHM_UNPROTECT();
7166 zend_jit_unprotect();
7167
7168 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7169
7170 t->id = ZEND_JIT_TRACE_NUM;
7171 t->root = ZEND_JIT_TRACE_NUM;
7172 t->parent = 0;
7173 t->link = 0;
7174 t->exit_count = 0;
7175 t->child_count = 0;
7176 t->stack_map_size = 0;
7177 t->flags = 0;
7178 t->polymorphism = 0;
7179 t->jmp_table_size = 0;
7180 t->op_array = trace_buffer[0].op_array;
7181 t->opline = trace_buffer[1].opline;
7182 t->exit_info = exit_info;
7183 t->stack_map = NULL;
7184
7185 orig_trigger = JIT_G(trigger);
7186 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7187
7188 handler = zend_jit_trace(trace_buffer, 0, 0);
7189
7190 JIT_G(trigger) = orig_trigger;
7191
7192 if (handler) {
7193 zend_jit_trace_exit_info *shared_exit_info = NULL;
7194
7195 t->exit_info = NULL;
7196 if (t->exit_count) {
7197 /* reallocate exit_info into shared memory */
7198 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7199 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7200
7201 if (!shared_exit_info) {
7202 if (t->stack_map) {
7203 efree(t->stack_map);
7204 t->stack_map = NULL;
7205 }
7206 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7207 goto exit;
7208 }
7209 memcpy(shared_exit_info, exit_info,
7210 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7211 t->exit_info = shared_exit_info;
7212 }
7213
7214 if (t->stack_map_size) {
7215 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7216 if (!shared_stack_map) {
7217 efree(t->stack_map);
7218 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7219 goto exit;
7220 }
7221 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7222 efree(t->stack_map);
7223 t->stack_map = shared_stack_map;
7224 }
7225
7226 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7227 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7228
7229 ((zend_op*)opline)->handler = handler;
7230
7231 ZEND_JIT_TRACE_NUM++;
7232 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7233
7234 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7235 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7236 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7237 if (t->stack_map) {
7238 efree(t->stack_map);
7239 t->stack_map = NULL;
7240 }
7241 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7242 } else {
7243 if (t->stack_map) {
7244 efree(t->stack_map);
7245 t->stack_map = NULL;
7246 }
7247 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7248 }
7249
7250 exit:;
7251 } zend_catch {
7252 do_bailout = 1;
7253 } zend_end_try();
7254
7255 zend_jit_protect();
7256 SHM_PROTECT();
7257 }
7258
7259 zend_shared_alloc_unlock();
7260
7261 if (do_bailout) {
7262 zend_bailout();
7263 }
7264
7265 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7266 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7267 && t->exit_count > 0) {
7268 zend_jit_dump_exit_info(t);
7269 }
7270
7271 return ret;
7272 }
7273
7274 /* Set counting handler back to original VM handler. */
zend_jit_stop_hot_trace_counters(zend_op_array * op_array)7275 static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array)
7276 {
7277 zend_jit_op_array_trace_extension *jit_extension;
7278 uint32_t i;
7279
7280 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7281 for (i = 0; i < op_array->last; i++) {
7282 /* Opline with Jit-ed code handler is skipped. */
7283 if (jit_extension->trace_info[i].trace_flags &
7284 (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7285 continue;
7286 }
7287 if (jit_extension->trace_info[i].trace_flags &
7288 (ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_START_RETURN)) {
7289 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7290 }
7291 }
7292 }
7293
7294 /* Get the tracing op_array. */
zend_jit_stop_persistent_op_array(zend_op_array * op_array)7295 static void zend_jit_stop_persistent_op_array(zend_op_array *op_array)
7296 {
7297 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
7298 if (!func_info) {
7299 return;
7300 }
7301 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
7302 zend_jit_stop_hot_trace_counters(op_array);
7303 }
7304 }
7305
7306 /* Get all op_arrays with counter handler. */
zend_jit_stop_persistent_script(zend_persistent_script * script)7307 static void zend_jit_stop_persistent_script(zend_persistent_script *script)
7308 {
7309 zend_class_entry *ce;
7310 zend_op_array *op_array;
7311
7312 zend_jit_stop_persistent_op_array(&script->script.main_op_array);
7313
7314 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
7315 zend_jit_stop_persistent_op_array(op_array);
7316 } ZEND_HASH_FOREACH_END();
7317
7318 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
7319 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
7320 if (op_array->type == ZEND_USER_FUNCTION) {
7321 zend_jit_stop_persistent_op_array(op_array);
7322 }
7323 } ZEND_HASH_FOREACH_END();
7324 } ZEND_HASH_FOREACH_END();
7325 }
7326
7327 /* Get all scripts which are accelerated by JIT */
zend_jit_stop_counter_handlers(void)7328 static void zend_jit_stop_counter_handlers(void)
7329 {
7330 if (ZCSG(jit_counters_stopped)) {
7331 return;
7332 }
7333
7334 zend_shared_alloc_lock();
7335 /* mprotect has an extreme overhead, avoid calls to it for every function. */
7336 SHM_UNPROTECT();
7337 if (!ZCSG(jit_counters_stopped)) {
7338 ZCSG(jit_counters_stopped) = true;
7339 for (uint32_t i = 0; i < ZCSG(hash).max_num_entries; i++) {
7340 zend_accel_hash_entry *cache_entry;
7341 for (cache_entry = ZCSG(hash).hash_table[i]; cache_entry; cache_entry = cache_entry->next) {
7342 zend_persistent_script *script;
7343 if (cache_entry->indirect) continue;
7344 script = (zend_persistent_script *)cache_entry->data;
7345 zend_jit_stop_persistent_script(script);
7346 }
7347 }
7348 }
7349 SHM_PROTECT();
7350 zend_shared_alloc_unlock();
7351 }
7352
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7353 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7354 {
7355 zend_shared_alloc_lock();
7356
7357 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7358 SHM_UNPROTECT();
7359 zend_jit_unprotect();
7360
7361 ((zend_op*)opline)->handler =
7362 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7363
7364 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7365
7366 zend_jit_protect();
7367 SHM_PROTECT();
7368 }
7369
7370 zend_shared_alloc_unlock();
7371 }
7372
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7373 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7374 {
7375 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7376 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7377 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7378 uint32_t cache_slot = JIT_G(bad_root_slot);
7379 uint32_t i;
7380
7381 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7382 if (cache_opline[i] == opline) {
7383 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7384 cache_opline[i] = NULL;
7385 return 1;
7386 } else {
7387 #if 0
7388 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7389 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7390 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7391 }
7392 #endif
7393 cache_count[i]++;
7394 cache_stop[i] = stop;
7395 return 0;
7396 }
7397 }
7398 }
7399 i = cache_slot;
7400 cache_opline[i] = opline;
7401 cache_count[i] = 1;
7402 cache_stop[i] = stop;
7403 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7404 JIT_G(bad_root_slot) = cache_slot;
7405 return 0;
7406 }
7407
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7408 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7409 {
7410 zend_jit_trace_rec *p = trace_buffer;
7411 const zend_op_array *op_array;
7412 const zend_op *opline;
7413 uint32_t level = 1 + trace_buffer[0].level;
7414 int idx, len, i, v, vars_count, call_level;
7415
7416 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7417 op_array = p->op_array;
7418 p += ZEND_JIT_TRACE_START_REC_SIZE;
7419 idx = 0;
7420 call_level = 0;
7421
7422 if (tssa && tssa->var_info) {
7423 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7424 vars_count = op_array->last_var;
7425 } else {
7426 vars_count = op_array->last_var + op_array->T;
7427 }
7428 for (i = 0; i < vars_count; i++) {
7429 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7430 fprintf(stderr, " %*c;", level, ' ');
7431 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7432 fprintf(stderr, "\n");
7433 }
7434 }
7435 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7436 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7437 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7438 zend_ssa_phi *p = tssa->blocks[1].phis;
7439
7440 fprintf(stderr, "LOOP:\n");
7441
7442 while (p) {
7443 fprintf(stderr, " ;");
7444 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7445 fprintf(stderr, " = Phi(");
7446 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7447 fprintf(stderr, ", ");
7448 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7449 fprintf(stderr, ")\n");
7450 p = p->next;
7451 }
7452 }
7453 }
7454
7455 while (1) {
7456 if (p->op == ZEND_JIT_TRACE_VM) {
7457 uint8_t op1_type, op2_type, op3_type;
7458
7459 opline = p->opline;
7460 fprintf(stderr, "%04d%*c",
7461 (int)(opline - op_array->opcodes),
7462 level, ' ');
7463 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7464
7465 op1_type = p->op1_type;
7466 op2_type = p->op2_type;
7467 op3_type = p->op3_type;
7468 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7469 fprintf(stderr, " ;");
7470 if (op1_type != IS_UNKNOWN) {
7471 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7472 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7473 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7474 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7475 p++;
7476 fprintf(stderr, " op1(%sobject of class %s)", ref,
7477 ZSTR_VAL(p->ce->name));
7478 } else {
7479 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7480 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7481 }
7482 }
7483 if (op2_type != IS_UNKNOWN) {
7484 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7485 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7486 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7487 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7488 p++;
7489 fprintf(stderr, " op2(%sobject of class %s)", ref,
7490 ZSTR_VAL(p->ce->name));
7491 } else {
7492 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7493 fprintf(stderr, " op2(%s%s)", ref, type);
7494 }
7495 }
7496 if (op3_type != IS_UNKNOWN) {
7497 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7498 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7499 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7500 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7501 fprintf(stderr, " op3(%s%s)", ref, type);
7502 }
7503 }
7504 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7505 uint8_t val_type;
7506 const char *type;
7507
7508 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7509 fprintf(stderr, " ;");
7510 }
7511 p++;
7512 val_type = p->op1_type;
7513
7514 if (val_type == IS_UNDEF) {
7515 type = "undef";
7516 } else if (val_type == IS_REFERENCE) {
7517 type = "ref";
7518 } else {
7519 type = zend_get_type_by_const(val_type);
7520 }
7521 fprintf(stderr, " val(%s)", type);
7522 }
7523 fprintf(stderr, "\n");
7524 idx++;
7525
7526 len = zend_jit_trace_op_len(opline);
7527 while (len > 1) {
7528 opline++;
7529 fprintf(stderr, "%04d%*c;",
7530 (int)(opline - op_array->opcodes),
7531 level, ' ');
7532 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7533 idx++;
7534 len--;
7535 fprintf(stderr, "\n");
7536 }
7537 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7538 op_array = p->op_array;
7539 fprintf(stderr, " %*c>enter %s%s%s\n",
7540 level, ' ',
7541 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7542 op_array->scope ? "::" : "",
7543 op_array->function_name ?
7544 ZSTR_VAL(op_array->function_name) :
7545 ZSTR_VAL(op_array->filename));
7546 level++;
7547 if (tssa && tssa->var_info) {
7548 call_level++;
7549 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7550 vars_count = op_array->last_var;
7551 for (i = 0; i < vars_count; i++, v++) {
7552 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7553 fprintf(stderr, " %*c;", level, ' ');
7554 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7555 fprintf(stderr, "\n");
7556 }
7557 }
7558 }
7559 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7560 op_array = p->op_array;
7561 level--;
7562 fprintf(stderr, " %*c<back %s%s%s\n",
7563 level, ' ',
7564 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7565 op_array->scope ? "::" : "",
7566 op_array->function_name ?
7567 ZSTR_VAL(op_array->function_name) :
7568 ZSTR_VAL(op_array->filename));
7569 if (tssa && tssa->var_info) {
7570 if (call_level == 0) {
7571 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7572 vars_count = op_array->last_var + op_array->T;
7573 for (i = 0; i < vars_count; i++, v++) {
7574 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7575 fprintf(stderr, " %*c;", level, ' ');
7576 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7577 fprintf(stderr, "\n");
7578 }
7579 }
7580 } else {
7581 call_level--;
7582 }
7583 }
7584 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7585 if (p->func != (zend_function*)&zend_pass_function) {
7586 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7587 level, ' ',
7588 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7589 (p->func && p->func->common.scope) ? "::" : "",
7590 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7591 } else {
7592 fprintf(stderr, " %*c>skip\n",
7593 level, ' ');
7594 }
7595 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7596 if (p->func != (zend_function*)&zend_pass_function) {
7597 fprintf(stderr, " %*c>call %s%s%s\n",
7598 level, ' ',
7599 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7600 p->func->common.scope ? "::" : "",
7601 ZSTR_VAL(p->func->common.function_name));
7602 } else {
7603 fprintf(stderr, " %*c>skip\n",
7604 level, ' ');
7605 }
7606 } else if (p->op == ZEND_JIT_TRACE_END) {
7607 break;
7608 }
7609 p++;
7610 }
7611 }
7612
zend_jit_dump_exit_info(zend_jit_trace_info * t)7613 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7614 {
7615 int i, j;
7616
7617 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7618 for (i = 0; i < t->exit_count; i++) {
7619 const zend_op_array *op_array = t->exit_info[i].op_array;
7620 uint32_t stack_size = t->exit_info[i].stack_size;
7621 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset;
7622
7623 fprintf(stderr, " exit_%d:", i);
7624 if (t->exit_info[i].opline) {
7625 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7626 } else {
7627 fprintf(stderr, " ----/");
7628 }
7629 if (t->exit_info[i].stack_size) {
7630 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7631 } else {
7632 fprintf(stderr, "----/0");
7633 }
7634 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7635 fprintf(stderr, "/VM");
7636 }
7637 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7638 fprintf(stderr, "/CALL");
7639 }
7640 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7641 fprintf(stderr, "/POLY");
7642 }
7643 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7644 fprintf(stderr, "/FREE_OP1");
7645 }
7646 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7647 fprintf(stderr, "/FREE_OP2");
7648 }
7649 for (j = 0; j < stack_size; j++) {
7650 uint8_t type = STACK_TYPE(stack, j);
7651 if (type != IS_UNKNOWN) {
7652 fprintf(stderr, " ");
7653 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7654 fprintf(stderr, ":");
7655 if (type == IS_UNDEF) {
7656 fprintf(stderr, "undef");
7657 } else {
7658 fprintf(stderr, "%s", zend_get_type_by_const(type));
7659 }
7660 if (STACK_REG(stack, j) != ZREG_NONE) {
7661 if (STACK_REG(stack, j) < ZREG_NUM) {
7662 fprintf(stderr, "(%s)", zend_reg_name[STACK_REG(stack, j)]);
7663 } else if (STACK_REG(stack, j) == ZREG_THIS) {
7664 fprintf(stderr, "(this)");
7665 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7666 fprintf(stderr, "(zval_try_addref)");
7667 } else {
7668 fprintf(stderr, "(const_%d)", STACK_REG(stack, j) - ZREG_NUM);
7669 }
7670 }
7671 } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) {
7672 fprintf(stderr, " ");
7673 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7674 fprintf(stderr, ":unknown(zval_try_addref)");
7675 } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_GPR0) {
7676 fprintf(stderr, " ");
7677 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7678 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[ZREG_COPY]);
7679 }
7680 }
7681 fprintf(stderr, "\n");
7682 }
7683 }
7684
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)7685 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
7686 {
7687 const zend_op *orig_opline;
7688 zend_jit_trace_stop stop;
7689 int ret = 0;
7690 zend_op_array *op_array;
7691 zend_jit_op_array_trace_extension *jit_extension;
7692 size_t offset;
7693 uint32_t trace_num;
7694 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7695
7696 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
7697 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
7698 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
7699
7700 repeat:
7701 trace_num = ZEND_JIT_TRACE_NUM;
7702 orig_opline = opline;
7703 op_array = &EX(func)->op_array;
7704 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7705 offset = jit_extension->offset;
7706
7707 EX(opline) = opline;
7708
7709 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
7710 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7711 return 0;
7712 }
7713
7714 if (JIT_G(tracing)) {
7715 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
7716 return 0;
7717 }
7718
7719 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7720 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7721 trace_num,
7722 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7723 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7724 EX(func)->op_array.scope ? "::" : "",
7725 EX(func)->op_array.function_name ?
7726 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7727 ZSTR_VAL(EX(func)->op_array.filename),
7728 opline->lineno);
7729 }
7730
7731 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7732 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7733 zend_jit_stop_counter_handlers();
7734 goto abort;
7735 }
7736
7737 JIT_G(tracing) = 1;
7738 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
7739 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
7740 JIT_G(tracing) = 0;
7741
7742 if (stop & ZEND_JIT_TRACE_HALT) {
7743 ret = -1;
7744 }
7745 stop &= ~ZEND_JIT_TRACE_HALT;
7746
7747 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
7748 orig_opline = trace_buffer[1].opline;
7749 op_array = (zend_op_array*)trace_buffer[0].op_array;
7750 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7751 offset = jit_extension->offset;
7752 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7753 const zend_op_array *op_array = trace_buffer[0].op_array;
7754 const zend_op *opline = trace_buffer[1].opline;
7755 zend_jit_op_array_trace_extension *jit_extension =
7756 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7757 size_t offset = jit_extension->offset;
7758
7759 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7760 trace_num,
7761 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7762 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7763 op_array->scope ? "::" : "",
7764 op_array->function_name ?
7765 ZSTR_VAL(op_array->function_name) : "$main",
7766 ZSTR_VAL(op_array->filename),
7767 opline->lineno);
7768 }
7769 }
7770
7771 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
7772 zend_jit_dump_trace(trace_buffer, NULL);
7773 }
7774
7775 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
7776 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
7777 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
7778 uint32_t idx = trace_buffer[1].last;
7779 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
7780 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
7781 trace_num,
7782 link_to);
7783 } else {
7784 fprintf(stderr, "---- TRACE %d stop (%s)\n",
7785 trace_num,
7786 zend_jit_trace_stop_description[stop]);
7787 }
7788 }
7789 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
7790 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
7791 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
7792 fprintf(stderr, "---- TRACE %d %s\n",
7793 trace_num,
7794 zend_jit_trace_stop_description[stop]);
7795 }
7796 } else {
7797 goto abort;
7798 }
7799 } else {
7800 abort:
7801 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
7802 fprintf(stderr, "---- TRACE %d abort (%s)\n",
7803 trace_num,
7804 zend_jit_trace_stop_description[stop]);
7805 }
7806 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
7807 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
7808 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7809 fprintf(stderr, "---- TRACE %d blacklisted\n",
7810 trace_num);
7811 }
7812 zend_jit_blacklist_root_trace(orig_opline, offset);
7813 }
7814 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
7815 execute_data = EG(current_execute_data);
7816 opline = EX(opline);
7817 goto repeat;
7818 }
7819 }
7820
7821 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
7822 fprintf(stderr, "\n");
7823 }
7824
7825 return ret;
7826 }
7827
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)7828 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
7829 {
7830 const void *handler;
7831 bool do_bailout = 0;
7832
7833 zend_shared_alloc_lock();
7834
7835 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
7836 SHM_UNPROTECT();
7837 zend_jit_unprotect();
7838
7839 zend_try {
7840 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
7841
7842 if (handler) {
7843 zend_jit_link_side_trace(
7844 zend_jit_traces[trace_num].code_start,
7845 zend_jit_traces[trace_num].code_size,
7846 zend_jit_traces[trace_num].jmp_table_size,
7847 exit_num,
7848 handler);
7849 }
7850 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
7851 } zend_catch {
7852 do_bailout = 1;
7853 } zend_end_try();
7854
7855 zend_jit_protect();
7856 SHM_PROTECT();
7857 }
7858
7859 zend_shared_alloc_unlock();
7860
7861 if (do_bailout) {
7862 zend_bailout();
7863 }
7864 }
7865
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)7866 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
7867 {
7868 uint8_t *counter = JIT_G(exit_counters) +
7869 zend_jit_traces[trace_num].exit_counters + exit_num;
7870
7871 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
7872 return 1;
7873 }
7874 (*counter)++;
7875 return 0;
7876 }
7877
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)7878 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
7879 {
7880 uint8_t *counter = JIT_G(exit_counters) +
7881 zend_jit_traces[trace_num].exit_counters + exit_num;
7882
7883 if (*counter + 1 >= JIT_G(hot_side_exit)) {
7884 return 1;
7885 }
7886 (*counter)++;
7887 return 0;
7888 }
7889
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)7890 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
7891 {
7892 zend_jit_trace_stop ret;
7893 const void *handler;
7894 uint8_t orig_trigger;
7895 zend_jit_trace_info *t;
7896 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7897 bool do_bailout = 0;
7898
7899 zend_shared_alloc_lock();
7900
7901 /* Checks under lock */
7902 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
7903 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7904 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7905 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7906 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
7907 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
7908 } else {
7909 SHM_UNPROTECT();
7910 zend_jit_unprotect();
7911
7912 zend_try {
7913 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7914
7915 t->id = ZEND_JIT_TRACE_NUM;
7916 t->root = zend_jit_traces[parent_num].root;
7917 t->parent = parent_num;
7918 t->link = 0;
7919 t->exit_count = 0;
7920 t->child_count = 0;
7921 t->stack_map_size = 0;
7922 t->flags = 0;
7923 t->polymorphism = polymorphism;
7924 t->jmp_table_size = 0;
7925 t->opline = NULL;
7926 t->exit_info = exit_info;
7927 t->stack_map = NULL;
7928
7929 orig_trigger = JIT_G(trigger);
7930 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7931
7932 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
7933
7934 JIT_G(trigger) = orig_trigger;
7935
7936 if (handler) {
7937 zend_jit_trace_exit_info *shared_exit_info = NULL;
7938
7939 t->exit_info = NULL;
7940 if (t->exit_count) {
7941 /* reallocate exit_info into shared memory */
7942 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7943 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7944
7945 if (!shared_exit_info) {
7946 if (t->stack_map) {
7947 efree(t->stack_map);
7948 t->stack_map = NULL;
7949 }
7950 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7951 goto exit;
7952 }
7953 memcpy(shared_exit_info, exit_info,
7954 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7955 t->exit_info = shared_exit_info;
7956 }
7957
7958 if (t->stack_map_size) {
7959 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7960 if (!shared_stack_map) {
7961 efree(t->stack_map);
7962 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7963 goto exit;
7964 }
7965 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7966 efree(t->stack_map);
7967 t->stack_map = shared_stack_map;
7968 }
7969
7970 zend_jit_link_side_trace(
7971 zend_jit_traces[parent_num].code_start,
7972 zend_jit_traces[parent_num].code_size,
7973 zend_jit_traces[parent_num].jmp_table_size,
7974 exit_num,
7975 handler);
7976
7977 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7978 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7979
7980 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
7981 ZEND_JIT_TRACE_NUM++;
7982 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
7983
7984 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7985 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7986 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7987 if (t->stack_map) {
7988 efree(t->stack_map);
7989 t->stack_map = NULL;
7990 }
7991 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7992 } else {
7993 if (t->stack_map) {
7994 efree(t->stack_map);
7995 t->stack_map = NULL;
7996 }
7997 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7998 }
7999
8000 exit:;
8001 } zend_catch {
8002 do_bailout = 1;
8003 } zend_end_try();
8004
8005 zend_jit_protect();
8006 SHM_PROTECT();
8007 }
8008
8009 zend_shared_alloc_unlock();
8010
8011 if (do_bailout) {
8012 zend_bailout();
8013 }
8014
8015 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
8016 && ret == ZEND_JIT_TRACE_STOP_COMPILED
8017 && t->exit_count > 0) {
8018 zend_jit_dump_exit_info(t);
8019 }
8020
8021 return ret;
8022 }
8023
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)8024 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
8025 {
8026 zend_jit_trace_stop stop;
8027 int ret = 0;
8028 uint32_t trace_num;
8029 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8030 uint32_t is_megamorphic = 0;
8031 uint32_t polymorphism = 0;
8032
8033 trace_num = ZEND_JIT_TRACE_NUM;
8034
8035 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8036 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8037 return 0;
8038 }
8039
8040 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8041 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
8042 trace_num, parent_num, exit_num,
8043 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8044 EX(func)->op_array.scope ? "::" : "",
8045 EX(func)->op_array.function_name ?
8046 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8047 ZSTR_VAL(EX(func)->op_array.filename),
8048 EX(opline)->lineno);
8049 }
8050
8051 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8052 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8053 goto abort;
8054 }
8055
8056 if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8057 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8058 goto abort;
8059 }
8060
8061 if (JIT_G(max_polymorphic_calls) > 0) {
8062 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
8063 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
8064 && EX(call))) {
8065 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
8066 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
8067 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
8068 } else if (!zend_jit_traces[parent_num].polymorphism) {
8069 polymorphism = 1;
8070 } else if (exit_num == 0) {
8071 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
8072 }
8073 }
8074 }
8075
8076 JIT_G(tracing) = 1;
8077 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
8078 JIT_G(tracing) = 0;
8079
8080 if (stop & ZEND_JIT_TRACE_HALT) {
8081 ret = -1;
8082 }
8083 stop &= ~ZEND_JIT_TRACE_HALT;
8084
8085 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8086 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8087 const zend_op_array *op_array = trace_buffer[0].op_array;
8088 const zend_op *opline = trace_buffer[1].opline;
8089 zend_jit_op_array_trace_extension *jit_extension =
8090 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8091 size_t offset = jit_extension->offset;
8092
8093 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8094 trace_num,
8095 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8096 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8097 op_array->scope ? "::" : "",
8098 op_array->function_name ?
8099 ZSTR_VAL(op_array->function_name) : "$main",
8100 ZSTR_VAL(op_array->filename),
8101 opline->lineno);
8102 }
8103 }
8104
8105 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8106 zend_jit_dump_trace(trace_buffer, NULL);
8107 }
8108
8109 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8110 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8111 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8112 uint32_t idx = trace_buffer[1].last;
8113 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8114 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8115 trace_num,
8116 link_to);
8117 } else {
8118 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8119 trace_num,
8120 zend_jit_trace_stop_description[stop]);
8121 }
8122 }
8123 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8124 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8125 } else {
8126 const zend_op_array *op_array = trace_buffer[0].op_array;
8127 zend_jit_op_array_trace_extension *jit_extension =
8128 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8129 const zend_op *opline = trace_buffer[1].opline;
8130
8131 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8132 }
8133 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8134 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8135 fprintf(stderr, "---- TRACE %d %s\n",
8136 trace_num,
8137 zend_jit_trace_stop_description[stop]);
8138 }
8139 } else {
8140 goto abort;
8141 }
8142 } else {
8143 abort:
8144 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8145 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8146 trace_num,
8147 zend_jit_trace_stop_description[stop]);
8148 }
8149 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8150 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8151 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8152 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8153 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8154 parent_num, exit_num);
8155 }
8156 }
8157 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8158 execute_data = EG(current_execute_data);
8159 return zend_jit_trace_hot_root(execute_data, EX(opline));
8160 }
8161 }
8162
8163 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8164 fprintf(stderr, "\n");
8165 }
8166
8167 return ret;
8168 }
8169
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8170 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8171 {
8172 uint32_t trace_num = EG(jit_trace_num);
8173 zend_execute_data *execute_data = EG(current_execute_data);
8174 const zend_op *orig_opline = EX(opline);
8175 const zend_op *opline;
8176 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8177 int repeat_last_opline = 0;
8178
8179 /* Deoptimization of VM stack state */
8180 uint32_t i;
8181 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8182 zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset;
8183
8184 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8185 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8186 call->prev_execute_data = EX(call);
8187 EX(call) = call;
8188 }
8189
8190 for (i = 0; i < stack_size; i++) {
8191 if (STACK_REG(stack, i) != ZREG_NONE) {
8192 if (STACK_TYPE(stack, i) == IS_LONG) {
8193 zend_long val;
8194
8195 if (STACK_REG(stack, i) < ZREG_NUM) {
8196 val = regs->gpr[STACK_REG(stack, i)];
8197 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN) {
8198 val = ZEND_LONG_MIN;
8199 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX) {
8200 val = ZEND_LONG_MAX;
8201 } else {
8202 ZEND_UNREACHABLE();
8203 }
8204 ZVAL_LONG(EX_VAR_NUM(i), val);
8205 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8206 double val;
8207
8208 if (STACK_REG(stack, i) < ZREG_NUM) {
8209 val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8210 } else if (STACK_REG(stack, i) == ZREG_LONG_MIN_MINUS_1) {
8211 val = (double)ZEND_LONG_MIN - 1.0;
8212 } else if (STACK_REG(stack, i) == ZREG_LONG_MAX_PLUS_1) {
8213 val = (double)ZEND_LONG_MAX + 1.0;
8214 } else {
8215 ZEND_UNREACHABLE();
8216 }
8217 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8218 } else if (STACK_REG(stack, i) == ZREG_THIS) {
8219 zend_object *obj = Z_OBJ(EX(This));
8220
8221 GC_ADDREF(obj);
8222 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8223 } else if (STACK_REG(stack, i) == ZREG_NULL) {
8224 ZVAL_NULL(EX_VAR_NUM(i));
8225 } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) {
8226 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8227 } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_GPR0) {
8228 zval *val = (zval*)regs->gpr[ZREG_COPY];
8229
8230 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8231 /* Undefined array index or property */
8232 repeat_last_opline = 1;
8233 } else {
8234 ZVAL_COPY(EX_VAR_NUM(i), val);
8235 }
8236 } else {
8237 ZEND_UNREACHABLE();
8238 }
8239 }
8240 }
8241
8242 if (repeat_last_opline) {
8243 EX(opline) = t->exit_info[exit_num].opline - 1;
8244 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8245 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8246 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8247 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8248 }
8249 return 1;
8250 }
8251
8252 opline = t->exit_info[exit_num].opline;
8253
8254 if (opline) {
8255 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8256 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8257 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8258 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8259 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8260 EX(opline) = opline-1;
8261 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8262 }
8263 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8264 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8265 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8266 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8267 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8268 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8269 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8270 EX(opline) = opline-1;
8271 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8272 }
8273 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8274 if (EG(exception)) {
8275 return 1;
8276 }
8277 }
8278 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8279 zend_function *func = (zend_function*)regs->gpr[ZREG_COPY];
8280
8281 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8282 zend_string_release_ex(func->common.function_name, 0);
8283 zend_free_trampoline(func);
8284 EX(opline) = opline;
8285 return 1;
8286 }
8287 }
8288
8289 /* Set VM opline to continue interpretation */
8290 EX(opline) = opline;
8291 }
8292
8293 if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) {
8294 return 1;
8295 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8296 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8297 return 0;
8298 }
8299
8300 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8301 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8302 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8303
8304 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8305 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8306 trace_num,
8307 exit_num,
8308 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8309 EX(func)->op_array.scope ? "::" : "",
8310 EX(func)->op_array.function_name ?
8311 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8312 ZSTR_VAL(EX(func)->op_array.filename),
8313 EX(opline)->lineno);
8314 }
8315
8316 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8317 zend_jit_op_array_trace_extension *jit_extension;
8318 uint32_t num = trace_num;
8319
8320 while (t->root != num) {
8321 num = t->root;
8322 t = &zend_jit_traces[num];
8323 }
8324
8325 zend_shared_alloc_lock();
8326
8327 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8328
8329 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8330 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8331 /* skip: not JIT-ed nor blacklisted */
8332 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8333 /* too many root traces, blacklist the root trace */
8334 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
8335 SHM_UNPROTECT();
8336 zend_jit_unprotect();
8337
8338 ((zend_op*)opline)->handler =
8339 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->orig_handler;
8340
8341 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= ~ZEND_JIT_TRACE_JITED;
8342 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
8343
8344 zend_jit_protect();
8345 SHM_PROTECT();
8346 }
8347 } else {
8348 SHM_UNPROTECT();
8349 zend_jit_unprotect();
8350
8351 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8352 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8353 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8354 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8355 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8356 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8357 }
8358 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8359 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8360
8361 zend_jit_protect();
8362 SHM_PROTECT();
8363 }
8364
8365 zend_shared_alloc_unlock();
8366
8367 return 0;
8368 }
8369
8370 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8371 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8372 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8373 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8374 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8375 trace_num, exit_num);
8376 }
8377 return 0;
8378 }
8379 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8380 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8381 }
8382
8383 /* Return 1 to call original handler instead of the same JIT-ed trace */
8384 return (orig_opline == t->opline && EX(opline) == orig_opline);
8385 }
8386
zend_jit_trace_supported(const zend_op * opline)8387 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8388 {
8389 switch (opline->opcode) {
8390 case ZEND_CATCH:
8391 case ZEND_FAST_CALL:
8392 case ZEND_FAST_RET:
8393 return ZEND_JIT_TRACE_UNSUPPORTED;
8394 default:
8395 return ZEND_JIT_TRACE_SUPPORTED;
8396 }
8397 }
8398
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8399 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8400 {
8401 zend_jit_op_array_trace_extension *jit_extension;
8402 uint32_t i;
8403
8404 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8405 for (i = 0; i < op_array->last; i++) {
8406 jit_extension->trace_info[i].trace_flags &=
8407 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8408 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8409 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8410 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8411 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8412 } else {
8413 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8414 }
8415 }
8416 return SUCCESS;
8417 }
8418
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8419 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8420 {
8421 zend_op *opline;
8422 zend_jit_op_array_trace_extension *jit_extension;
8423 uint32_t i;
8424
8425 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8426
8427 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8428 if (!jit_extension) {
8429 return FAILURE;
8430 }
8431 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8432 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8433 jit_extension->op_array = op_array;
8434 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8435 for (i = 0; i < op_array->last; i++) {
8436 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8437 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8438 jit_extension->trace_info[i].counter = NULL;
8439 jit_extension->trace_info[i].trace_flags =
8440 zend_jit_trace_supported(&op_array->opcodes[i]);
8441 }
8442 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8443
8444 if (JIT_G(hot_loop)) {
8445 zend_cfg cfg;
8446
8447 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8448
8449 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8450 return FAILURE;
8451 }
8452
8453 for (i = 0; i < cfg.blocks_count; i++) {
8454 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8455 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8456 /* loop header */
8457 opline = op_array->opcodes + cfg.blocks[i].start;
8458 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8459 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8460 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8461 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8462 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8463 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8464 }
8465 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8466 ZEND_JIT_TRACE_START_LOOP;
8467 }
8468 }
8469 }
8470 }
8471 }
8472
8473 if (JIT_G(hot_func)) {
8474 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8475 opline = op_array->opcodes;
8476 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8477 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8478 opline++;
8479 }
8480 }
8481
8482 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8483 /* function entry */
8484 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8485 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8486 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8487 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8488 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8489 ZEND_JIT_TRACE_START_ENTER;
8490 }
8491 }
8492
8493 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8494
8495 return SUCCESS;
8496 }
8497
zend_jit_trace_init_caches(void)8498 static void zend_jit_trace_init_caches(void)
8499 {
8500 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8501 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8502 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8503 JIT_G(bad_root_slot) = 0;
8504
8505 if (JIT_G(exit_counters)) {
8506 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8507 }
8508 }
8509
zend_jit_trace_reset_caches(void)8510 static void zend_jit_trace_reset_caches(void)
8511 {
8512 JIT_G(tracing) = 0;
8513 #ifdef ZTS
8514 if (!JIT_G(exit_counters)) {
8515 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8516 }
8517 #endif
8518 }
8519
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8520 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8521 {
8522 if (jit_globals->exit_counters) {
8523 free(jit_globals->exit_counters);
8524 }
8525 }
8526
zend_jit_trace_restart(void)8527 static void zend_jit_trace_restart(void)
8528 {
8529 ZEND_JIT_TRACE_NUM = 1;
8530 ZEND_JIT_COUNTER_NUM = 0;
8531 ZEND_JIT_EXIT_NUM = 0;
8532 ZEND_JIT_EXIT_COUNTERS = 0;
8533 ZCSG(jit_counters_stopped) = false;
8534
8535 zend_jit_trace_init_caches();
8536 }
8537
