1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(bool reattached)49 static void zend_jit_trace_startup(bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT root traces buffer!");
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT exit groups buffer!");
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 ZCSG(jit_counters_stopped) = false;
67 } else {
68 zend_jit_traces = ZCSG(jit_traces);
69 if (!zend_jit_traces) {
70 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not obtain JIT traces buffer!");
71 }
72 zend_jit_exit_groups = ZCSG(jit_exit_groups);
73 if (!zend_jit_exit_groups) {
74 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not obtain JIT exit groups buffer!");
75 }
76 }
77
78 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
79 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
80
81 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
82 if (JIT_G(exit_counters) == NULL) {
83 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT exit counters buffer!");
84 }
85 }
86
zend_jit_trace_allocate_exit_point(uint32_t n)87 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
88 {
89 const void *group = NULL;
90
91 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
92 return NULL;
93 }
94 do {
95 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
96 if (!group) {
97 return NULL;
98 }
99 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
100 group;
101 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
102 } while (n >= ZEND_JIT_EXIT_NUM);
103 return (const void*)
104 ((const char*)group +
105 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
106 }
107
zend_jit_trace_get_exit_addr(uint32_t n)108 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
109 {
110 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
111 return zend_jit_trace_allocate_exit_point(n);
112 }
113 return (const void*)
114 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
115 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
116 }
117
zend_jit_exit_point_by_addr(const void * addr)118 static uint32_t zend_jit_exit_point_by_addr(const void *addr)
119 {
120 uint32_t n = (ZEND_JIT_EXIT_NUM + (ZEND_JIT_EXIT_POINTS_PER_GROUP - 1)) / ZEND_JIT_EXIT_POINTS_PER_GROUP;
121 uint32_t i;
122
123 for (i = 0; i < n; i++) {
124 if ((char*)addr >= (char*)zend_jit_exit_groups[i]
125 && (char*)addr <= (char*)zend_jit_exit_groups[i] + ((ZEND_JIT_EXIT_POINTS_PER_GROUP - 1) * ZEND_JIT_EXIT_POINTS_SPACING)) {
126 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
127 (((char*)addr - (char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING);
128 }
129 }
130 return (uint32_t)-1;
131 }
132
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)133 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
134 {
135 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
136 uint32_t exit_point;
137 const zend_op_array *op_array;
138 uint32_t stack_offset = (uint32_t)-1;
139 uint32_t stack_size;
140 zend_jit_trace_stack *stack = NULL;
141
142 if (delayed_call_chain) {
143 assert(to_opline != NULL); /* CALL and IP share the same register */
144 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
145 }
146 if (JIT_G(current_frame)) {
147 op_array = &JIT_G(current_frame)->func->op_array;
148 stack_size = op_array->last_var + op_array->T;
149 if (stack_size) {
150 stack = JIT_G(current_frame)->stack;
151 do {
152 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
153 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
154 || STACK_REF(stack, stack_size-1) != IR_UNUSED
155 ) {
156 break;
157 }
158 stack_size--;
159 } while (stack_size);
160 }
161 } else {
162 op_array = NULL;
163 stack_size = 0;
164 }
165
166 /* Try to reuse exit points */
167 if (to_opline != NULL
168 && !(flags & ZEND_JIT_EXIT_METHOD_CALL)
169 && t->exit_count > 0) {
170 uint32_t i = t->exit_count;
171
172 do {
173 i--;
174 if (stack_size == 0
175 || (t->exit_info[i].stack_size >= stack_size
176 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
177 if (t->exit_info[i].opline == to_opline
178 && t->exit_info[i].flags == flags
179 && t->exit_info[i].stack_size == stack_size) {
180 return i;
181 }
182 }
183 } while (i > 0);
184 }
185
186 exit_point = t->exit_count;
187 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
188 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
189 stack_offset = t->stack_map_size;
190 t->stack_map_size += stack_size;
191 // TODO: reduce number of reallocations ???
192 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
193 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
194 }
195 t->exit_count++;
196 t->exit_info[exit_point].opline = to_opline;
197 t->exit_info[exit_point].op_array = op_array;
198 t->exit_info[exit_point].flags = flags;
199 t->exit_info[exit_point].stack_size = stack_size;
200 t->exit_info[exit_point].stack_offset = stack_offset;
201 t->exit_info[exit_point].poly_func_ref = 0;
202 t->exit_info[exit_point].poly_this_ref = 0;
203 t->exit_info[exit_point].poly_func_reg = ZREG_NONE;
204 t->exit_info[exit_point].poly_this_reg = ZREG_NONE;
205 }
206
207 return exit_point;
208 }
209
zend_jit_trace_add_code(const void * start,uint32_t size)210 static void zend_jit_trace_add_code(const void *start, uint32_t size)
211 {
212 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
213
214 t->code_start = start;
215 t->code_size = size;
216 }
217
218 /**
219 * Locate a trace in the #zend_jit_traces array with the specified
220 * #code_start address.
221 *
222 * @return the #zend_jit_traces index or 0 if no such #code_start
223 * address was found
224 */
zend_jit_find_trace(const void * addr)225 static uint32_t zend_jit_find_trace(const void *addr)
226 {
227 uint32_t i;
228
229 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
230 if (zend_jit_traces[i].code_start == addr) {
231 return i;
232 }
233 }
234 return 0;
235 }
236
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)237 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
238 {
239 smart_str buf = {0};
240
241 smart_str_appends(&buf, TRACE_PREFIX);
242 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
243 smart_str_appendc(&buf, '$');
244 if (op_array->function_name) {
245 if (op_array->scope) {
246 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
247 smart_str_appends(&buf, "::");
248 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
249 } else {
250 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
251 }
252 } else if (op_array->filename) {
253 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
254 }
255 smart_str_appendc(&buf, '$');
256 smart_str_append_long(&buf, (zend_long)lineno);
257 smart_str_0(&buf);
258 return buf.s;
259 }
260
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)261 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
262 {
263 switch (opline->opcode) {
264 case ZEND_IS_IDENTICAL:
265 case ZEND_IS_NOT_IDENTICAL:
266 case ZEND_IS_EQUAL:
267 case ZEND_IS_NOT_EQUAL:
268 case ZEND_IS_SMALLER:
269 case ZEND_IS_SMALLER_OR_EQUAL:
270 case ZEND_CASE:
271 case ZEND_CASE_STRICT:
272 case ZEND_ISSET_ISEMPTY_CV:
273 case ZEND_ISSET_ISEMPTY_VAR:
274 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
275 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
276 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
277 case ZEND_INSTANCEOF:
278 case ZEND_TYPE_CHECK:
279 case ZEND_DEFINED:
280 case ZEND_IN_ARRAY:
281 case ZEND_ARRAY_KEY_EXISTS:
282 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
283 /* smart branch */
284 return 1;
285 }
286 break;
287 case ZEND_JMPZ:
288 case ZEND_JMPNZ:
289 case ZEND_JMPZ_EX:
290 case ZEND_JMPNZ_EX:
291 case ZEND_JMP_SET:
292 case ZEND_COALESCE:
293 case ZEND_JMP_NULL:
294 case ZEND_FE_RESET_R:
295 case ZEND_FE_RESET_RW:
296 case ZEND_ASSERT_CHECK:
297 case ZEND_FE_FETCH_R:
298 case ZEND_FE_FETCH_RW:
299 case ZEND_SWITCH_LONG:
300 case ZEND_SWITCH_STRING:
301 case ZEND_MATCH:
302 case ZEND_BIND_INIT_STATIC_OR_JMP:
303 case ZEND_JMP_FRAMELESS:
304 /* branch opcodes */
305 return 1;
306 case ZEND_NEW:
307 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
308 /* NEW may skip constructor without arguments */
309 return 1;
310 }
311 break;
312 case ZEND_CATCH:
313 case ZEND_FAST_CALL:
314 case ZEND_FAST_RET:
315 case ZEND_GENERATOR_CREATE:
316 case ZEND_GENERATOR_RETURN:
317 case ZEND_YIELD:
318 case ZEND_YIELD_FROM:
319 case ZEND_INCLUDE_OR_EVAL:
320 case ZEND_MATCH_ERROR:
321 /* unsupported */
322 return 1;
323 case ZEND_DO_FCALL:
324 /* potentially polymorphic call */
325 return 1;
326 #if 0
327 case ZEND_DO_UCALL:
328 case ZEND_DO_FCALL_BY_NAME:
329 /* monomorphic call */
330 // TODO: recompilation may change target ???
331 return 0;
332 #endif
333 case ZEND_RETURN_BY_REF:
334 case ZEND_RETURN:
335 /* return */
336 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
337 default:
338 break;
339 }
340 return 0;
341 }
342
zend_jit_trace_type_to_info_ex(uint8_t type,uint32_t info)343 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(uint8_t type, uint32_t info)
344 {
345 if (type == IS_UNKNOWN) {
346 return info;
347 }
348 ZEND_ASSERT(info & (1 << type));
349 if (type < IS_STRING) {
350 return (1 << type);
351 } else if (type != IS_ARRAY) {
352 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
353 } else {
354 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
355 }
356 }
357
zend_jit_trace_type_to_info(uint8_t type)358 static zend_always_inline uint32_t zend_jit_trace_type_to_info(uint8_t type)
359 {
360 return zend_jit_trace_type_to_info_ex(type, -1);
361 }
362
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)363 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
364 {
365 if (var >= op_array->last_var) {
366 return NO_ALIAS;
367 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
368 return SYMTABLE_ALIAS;
369 } else if (ssa->vars) {
370 return ssa->vars[var].alias;
371 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
372 return HTTP_RESPONSE_HEADER_ALIAS;
373 }
374 return NO_ALIAS;
375 }
376
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)377 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
378 int ssa_var,
379 uint8_t op_type)
380 {
381 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
382
383 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
384 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
385 info->type |= MAY_BE_GUARD;
386 } else {
387 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
388 }
389 }
390 }
391
392 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
393 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
394 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
395 } \
396 } while (0)
397
398 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
399 if (op_type != IS_UNKNOWN) { \
400 if ((op_info & MAY_BE_GUARD) != 0) { \
401 if (!zend_jit_type_guard(&ctx, opline, _var, op_type)) { \
402 goto jit_failure; \
403 } \
404 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
405 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
406 op_info = zend_jit_trace_type_to_info(op_type); \
407 } else { \
408 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
409 op_info &= ~MAY_BE_GUARD; \
410 ssa->var_info[_ssa_var].type &= op_info; \
411 } \
412 } \
413 } \
414 } while (0)
415
416 #define ADD_OP1_TRACE_GUARD() \
417 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
418 #define ADD_OP2_TRACE_GUARD() \
419 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
420 #define ADD_OP1_DATA_TRACE_GUARD() \
421 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
422
423 #define CHECK_OP1_TRACE_TYPE() \
424 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
425 #define CHECK_OP2_TRACE_TYPE() \
426 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
427 #define CHECK_OP1_DATA_TRACE_TYPE() \
428 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
429
zend_jit_trace_frame_size(const zend_op_array * op_array)430 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
431 {
432 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
433 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
434 } else if (op_array) {
435 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
436 } else {
437 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
438 }
439 }
440
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)441 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
442 {
443 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
444 }
445
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)446 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
447 {
448 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
449 }
450
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,uint8_t type)451 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type)
452 {
453 zend_jit_trace_stack *stack = call->stack;
454 const zend_op_array *op_array = &call->func->op_array;
455 uint32_t arg_num = opline->op2.num;
456
457 if (arg_num > op_array->num_args) {
458 return;
459 }
460 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
461 zend_arg_info *arg_info;
462
463 ZEND_ASSERT(arg_num <= op_array->num_args);
464 arg_info = &op_array->arg_info[arg_num-1];
465
466 if (ZEND_TYPE_IS_SET(arg_info->type)) {
467 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
468 return;
469 }
470 }
471 }
472 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
473 }
474
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)475 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
476 {
477 if (func
478 && func->type == ZEND_INTERNAL_FUNCTION
479 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
480 && arg_num < func->internal_function.num_args) {
481 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
482
483 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
484 && ZEND_TYPE_IS_SET(arg_info->type)
485 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
486 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
487 && JIT_G(current_frame)
488 && JIT_G(current_frame)->call
489 && JIT_G(current_frame)->call->func) {
490 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
491
492 if (type != IS_UNKNOWN
493 && type < IS_STRING
494 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
495 return 0;
496 }
497 }
498 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
499 const zend_op *opline = call_info->arg_info[arg_num].opline;
500
501 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
502 zval *zv = RT_CONSTANT(opline, opline->op1);
503
504 if (!Z_REFCOUNTED_P(zv)) {
505 uint32_t type = Z_TYPE_P(zv);
506
507 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
508 if (type != IS_ARRAY
509 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
510 return 0;
511 }
512 }
513 }
514 }
515 }
516 }
517
518 return 1;
519 }
520
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)521 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
522 {
523 zend_jit_op_array_trace_extension *jit_extension;
524 zend_ssa *ssa;
525
526 jit_extension =
527 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
528 jit_extension->func_info.num = 0;
529 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
530 | ZEND_FUNC_JIT_ON_PROF_REQUEST
531 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
532 | ZEND_FUNC_JIT_ON_HOT_TRACE;
533 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
534 ssa = &jit_extension->func_info.ssa;
535
536 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
537 do {
538 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
539 break;
540 }
541
542 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
543 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info);
544 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
545 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
546 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
547 }
548 }
549
550 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
551 break;
552 }
553
554 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
555 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
556 }
557 return ssa;
558 } while (0);
559 }
560
561 memset(ssa, 0, sizeof(zend_ssa));
562 ssa->cfg.blocks_count = 1;
563
564 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
565 zend_cfg cfg;
566 void *checkpoint = zend_arena_checkpoint(CG(arena));
567
568 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
569 ssa->cfg.flags = cfg.flags;
570 } else{
571 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
572 }
573
574 /* TODO: move this to zend_cfg.c ? */
575 if (!op_array->function_name) {
576 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
577 }
578
579 zend_arena_release(&CG(arena), checkpoint);
580 }
581
582 return ssa;
583 }
584
585 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
586 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
587
zend_jit_trace_op_len(const zend_op * opline)588 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
589 {
590 int len;
591
592 switch (opline->opcode) {
593 case ZEND_ASSIGN_DIM:
594 case ZEND_ASSIGN_OBJ:
595 case ZEND_ASSIGN_STATIC_PROP:
596 case ZEND_ASSIGN_DIM_OP:
597 case ZEND_ASSIGN_OBJ_OP:
598 case ZEND_ASSIGN_STATIC_PROP_OP:
599 case ZEND_ASSIGN_OBJ_REF:
600 case ZEND_ASSIGN_STATIC_PROP_REF:
601 case ZEND_FRAMELESS_ICALL_3:
602 return 2; /* OP_DATA */
603 case ZEND_RECV_INIT:
604 len = 1;
605 opline++;
606 while (opline->opcode == ZEND_RECV_INIT) {
607 len++;
608 opline++;
609 }
610 return len;
611 case ZEND_BIND_GLOBAL:
612 len = 1;
613 opline++;
614 while (opline->opcode == ZEND_BIND_GLOBAL) {
615 len++;
616 opline++;
617 }
618 return len;
619 // case ZEND_IS_IDENTICAL:
620 // case ZEND_IS_NOT_IDENTICAL:
621 // case ZEND_IS_EQUAL:
622 // case ZEND_IS_NOT_EQUAL:
623 // case ZEND_IS_SMALLER:
624 // case ZEND_IS_SMALLER_OR_EQUAL:
625 // case ZEND_CASE:
626 // case ZEND_ISSET_ISEMPTY_CV:
627 // case ZEND_ISSET_ISEMPTY_VAR:
628 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
629 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
630 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
631 // case ZEND_INSTANCEOF:
632 // case ZEND_TYPE_CHECK:
633 // case ZEND_DEFINED:
634 // case ZEND_IN_ARRAY:
635 // case ZEND_ARRAY_KEY_EXISTS:
636 default:
637 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
638 return 2; /* JMPZ/JMPNZ */
639 }
640 return 1;
641 }
642 }
643
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)644 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
645 {
646 const zend_op_array *op_array;
647 zend_jit_trace_rec *p;
648 int k, vars_count;
649 zend_bitset use, def;
650 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
651 uint32_t set_size;
652 zend_ssa_phi *prev = NULL;
653 int level = 0;
654 ALLOCA_FLAG(use_heap);
655
656 op_array = trace_buffer->op_array;
657 set_size = zend_bitset_len(op_array->last_var + op_array->T);
658 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
659 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
660 def = use + set_size;
661 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
662 for (;;p++) {
663 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
664 const zend_op *opline = p->opline;
665 int len;
666
667 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
668 len = zend_jit_trace_op_len(opline);
669 while (len > 1) {
670 opline++;
671 if (opline->opcode != ZEND_OP_DATA) {
672 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
673 }
674 len--;
675 }
676 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
677 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
678 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
679 level++;
680 } else if (p->op == ZEND_JIT_TRACE_BACK) {
681 if (level == 0) {
682 // Phi for recursive calls and returns are not supported yet ???
683 assert(0);
684 } else {
685 level--;
686 }
687 } else if (p->op == ZEND_JIT_TRACE_END) {
688 break;
689 }
690 }
691
692 zend_bitset_intersection(use, def, set_size);
693
694 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
695 vars_count = op_array->last_var;
696 } else {
697 vars_count = op_array->last_var + op_array->T;
698 }
699 for (k = 0; k < vars_count; k++) {
700 if (zend_bitset_in(use, k)) {
701 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
702 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
703 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
704 sizeof(void*) * 2);
705 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
706 phi->sources[0] = STACK_VAR(stack, k);
707 phi->sources[1] = -1;
708 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
709 phi->pi = -1;
710 phi->var = k;
711 phi->ssa_var = ssa_vars_count;
712 SET_STACK_VAR(stack, k, ssa_vars_count);
713 ssa_vars_count++;
714 phi->block = 1;
715 if (prev) {
716 prev->next = phi;
717 } else {
718 tssa->blocks[1].phis = phi;
719 }
720 prev = phi;
721 }
722 }
723
724 free_alloca(use, use_heap);
725
726 return ssa_vars_count;
727 }
728
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)729 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
730 {
731 zend_ssa_phi *prev = NULL;
732 const zend_op_array *op_array = trace_buffer->op_array;
733 const zend_op *opline = trace_buffer[1].opline;
734 int count = opline - op_array->opcodes;
735 int i;
736
737 for(i = 0; i < count; i++) {
738 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
739 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
740 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
741 sizeof(void*) * 2);
742 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
743 phi->sources[0] = STACK_VAR(stack, i);
744 phi->sources[1] = -1;
745 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
746 phi->pi = -1;
747 phi->var = i;
748 phi->ssa_var = ssa_vars_count;
749 SET_STACK_VAR(stack, i, ssa_vars_count);
750 ssa_vars_count++;
751 phi->block = 1;
752 if (prev) {
753 prev->next = phi;
754 } else {
755 tssa->blocks[1].phis = phi;
756 }
757 prev = phi;
758 }
759 return ssa_vars_count;
760 }
761
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)762 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
763 {
764 const zend_op *opline = trace_buffer[1].opline - 1;
765 int i;
766
767 if (RETURN_VALUE_USED(opline)) {
768 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
769 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
770 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
771 sizeof(void*) * 2);
772
773 i = EX_VAR_TO_NUM(opline->result.var);
774 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
775 phi->sources[0] = STACK_VAR(stack, i);
776 phi->sources[1] = -1;
777 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
778 phi->pi = -1;
779 phi->var = i;
780 phi->ssa_var = ssa_vars_count;
781 SET_STACK_VAR(stack, i, ssa_vars_count);
782 ssa_vars_count++;
783 phi->block = 1;
784 tssa->blocks[1].phis = phi;
785 }
786 return ssa_vars_count;
787 }
788
zend_jit_trace_is_false_loop(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa)789 static bool zend_jit_trace_is_false_loop(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa)
790 {
791 const zend_op *opline;
792 uint32_t b;
793 zend_basic_block *bb;
794
795 ZEND_ASSERT(tssa->cfg.blocks_count == 2);
796 ZEND_ASSERT(tssa->cfg.blocks[1].len > 0);
797
798 b = ssa->cfg.map[tssa_opcodes[0] - op_array->opcodes];
799 opline = tssa_opcodes[tssa->cfg.blocks[1].len - 1];
800 if (opline >= op_array->opcodes && opline < op_array->opcodes + op_array->last) {
801 bb = ssa->cfg.blocks + ssa->cfg.map[opline - op_array->opcodes];
802 return bb->loop_header != b;
803 } else {
804 return 0;
805 }
806 }
807
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var,const zend_op * opline)808 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array,
809 const zend_ssa *ssa,
810 const zend_op **tssa_opcodes,
811 zend_ssa *tssa,
812 int ssa_var,
813 const zend_op *opline)
814 {
815 int var, use, def, src;
816 zend_ssa_op *op;
817
818 if (tssa->vars[ssa_var].definition_phi) {
819 uint32_t b = ssa->cfg.map[tssa_opcodes[0] - op_array->opcodes];
820 zend_basic_block *bb = ssa->cfg.blocks + b;
821
822 if ((bb->flags & ZEND_BB_LOOP_HEADER)
823 && !zend_jit_trace_is_false_loop(op_array, ssa, tssa_opcodes, tssa)) {
824 zend_ssa_phi *phi = ssa->blocks[b].phis;
825 zend_ssa_phi *pi = NULL;
826
827 var = tssa->vars[ssa_var].var;
828 while (phi) {
829 if (ssa->vars[phi->ssa_var].var == var) {
830 if (phi->pi >= 0) {
831 pi = phi;
832 } else {
833 src = phi->ssa_var;
834 goto copy_info;
835 }
836 }
837 phi = phi->next;
838 }
839 if (pi) {
840 src = pi->ssa_var;
841 goto copy_info;
842 }
843 #if 0
844 while (bb->idom >= 0) {
845 uint32_t n;
846
847 b = bb->idom;
848 bb = ssa->cfg.blocks + b;
849
850 for (n = bb->len, op = ssa->ops + bb->start + n; n > 0; n--) {
851 op--;
852 if (op->result_def >= 0 && ssa->vars[op->result_def].var == var) {
853 src = op->result_def;
854 goto copy_info;
855 } else if (op->op2_def >= 0 && ssa->vars[op->op2_def].var == var) {
856 src = op->op2_def;
857 goto copy_info;
858 } else if (op->op1_def >= 0 && ssa->vars[op->op1_def].var == var) {
859 src = op->op1_def;
860 goto copy_info;
861 }
862 }
863
864 phi = ssa->blocks[b].phis;
865 zend_ssa_phi *pi = NULL;
866 while (phi) {
867 if (ssa->vars[phi->ssa_var].var == var) {
868 if (phi->pi >= 0) {
869 pi = phi;
870 } else {
871 src = phi->ssa_var;
872 goto copy_info;
873 }
874 }
875 phi = phi->next;
876 }
877 if (pi) {
878 src = pi->ssa_var;
879 goto copy_info;
880 }
881 }
882 #endif
883 }
884 } else if (tssa->vars[ssa_var].definition >= 0) {
885 def = tssa->vars[ssa_var].definition;
886 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
887 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
888 if (tssa->ops[def].op1_def == ssa_var) {
889 src = op->op1_def;
890 } else if (tssa->ops[def].op2_def == ssa_var) {
891 src = op->op2_def;
892 } else if (tssa->ops[def].result_def == ssa_var) {
893 src = op->result_def;
894 } else {
895 assert(0);
896 return 0;
897 }
898 goto copy_info;
899 }
900
901 if (tssa->vars[ssa_var].phi_use_chain) {
902 // TODO: this may be incorrect ???
903 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
904 } else {
905 var = ssa_var;
906 }
907 use = tssa->vars[var].use_chain;
908 if (use >= 0) {
909 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
910 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
911 if (tssa->ops[use].op1_use == var) {
912 src = op->op1_use;
913 } else if (tssa->ops[use].op2_use == var) {
914 src = op->op2_use;
915 } else if (tssa->ops[use].result_use == var) {
916 src = op->result_use;
917 } else {
918 assert(0);
919 return 0;
920 }
921 if (opline) {
922 /* Try to find a difinition in SSA dominators tree */
923 var = tssa->vars[ssa_var].var;
924 uint32_t op_num = opline - op_array->opcodes;
925 uint32_t b = ssa->cfg.map[op_num];
926 zend_basic_block *bb = ssa->cfg.blocks + b;
927 zend_ssa_phi *pi, *phi;
928
929 while (1) {
930 while (op_num > bb->start) {
931 op_num--;
932 op = ssa->ops + op_num;
933 if (op->result_def >= 0 && ssa->vars[op->result_def].var == var) {
934 src = op->result_def;
935 goto copy_info;
936 } else if (op->op2_def >= 0 && ssa->vars[op->op2_def].var == var) {
937 src = op->op2_def;
938 goto copy_info;
939 } else if (op->op1_def >= 0 && ssa->vars[op->op1_def].var == var) {
940 src = op->op1_def;
941 goto copy_info;
942 }
943 }
944 phi = ssa->blocks[b].phis;
945 pi = NULL;
946 while (phi) {
947 if (ssa->vars[phi->ssa_var].var == var) {
948 if (phi->pi >= 0) {
949 pi = phi;
950 } else {
951 src = phi->ssa_var;
952 goto copy_info;
953 }
954 }
955 phi = phi->next;
956 }
957 if (pi) {
958 src = pi->ssa_var;
959 goto copy_info;
960 }
961 if (bb->idom < 0) {
962 break;
963 }
964 b = bb->idom;
965 bb = ssa->cfg.blocks + b;
966 op_num = bb->start + bb->len;
967 }
968 }
969 goto copy_info;
970 }
971 return 0;
972
973 copy_info:
974 tssa->vars[ssa_var].no_val = ssa->vars[src].no_val;
975 tssa->vars[ssa_var].alias = ssa->vars[src].alias;
976 memcpy(&tssa->var_info[ssa_var], &ssa->var_info[src], sizeof(zend_ssa_var_info));
977 return 1;
978 }
979
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)980 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
981 {
982 zend_ssa_range tmp;
983 int def = tssa->vars[ssa_var].definition;
984
985 if (tssa->vars[ssa_var].alias == NO_ALIAS
986 && zend_inference_propagate_range(op_array, tssa, tssa_opcodes[def], &tssa->ops[def], ssa_var, &tmp)) {
987 tssa->var_info[ssa_var].range.min = tmp.min;
988 tssa->var_info[ssa_var].range.max = tmp.max;
989 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
990 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
991 tssa->var_info[ssa_var].has_range = 1;
992 }
993 }
994
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)995 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
996 {
997 int def;
998 zend_ssa_op *op;
999 zend_ssa_var_info *info;
1000 unsigned int no_val;
1001 zend_ssa_alias_kind alias;
1002
1003 def = tssa->vars[ssa_var].definition;
1004 if (def >= 0) {
1005 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
1006 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
1007 if (tssa->ops[def].op1_def == ssa_var) {
1008 no_val = ssa->vars[op->op1_def].no_val;
1009 alias = ssa->vars[op->op1_def].alias;
1010 info = ssa->var_info + op->op1_def;
1011 } else if (tssa->ops[def].op2_def == ssa_var) {
1012 no_val = ssa->vars[op->op2_def].no_val;
1013 alias = ssa->vars[op->op2_def].alias;
1014 info = ssa->var_info + op->op2_def;
1015 } else if (tssa->ops[def].result_def == ssa_var) {
1016 no_val = ssa->vars[op->result_def].no_val;
1017 alias = ssa->vars[op->result_def].alias;
1018 info = ssa->var_info + op->result_def;
1019 } else {
1020 assert(0);
1021 return;
1022 }
1023
1024 tssa->vars[ssa_var].no_val = no_val;
1025 tssa->vars[ssa_var].alias = alias;
1026
1027 if (!(info->type & MAY_BE_REF)) {
1028 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
1029 }
1030
1031 if (info->has_range) {
1032 if (tssa->var_info[ssa_var].has_range) {
1033 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
1034 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
1035 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
1036 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
1037 } else {
1038 tssa->var_info[ssa_var].has_range = 1;
1039 tssa->var_info[ssa_var].range = info->range;
1040 }
1041 }
1042 }
1043 }
1044
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)1045 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
1046 {
1047 int def;
1048 zend_ssa_op *op;
1049 zend_ssa_var_info *info;
1050
1051 def = tssa->vars[ssa_var].definition;
1052 if (def >= 0) {
1053 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
1054 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
1055 if (tssa->ops[def].op1_def == ssa_var) {
1056 info = ssa->var_info + op->op1_def;
1057 } else if (tssa->ops[def].op2_def == ssa_var) {
1058 info = ssa->var_info + op->op2_def;
1059 } else if (tssa->ops[def].result_def == ssa_var) {
1060 info = ssa->var_info + op->result_def;
1061 } else {
1062 assert(0);
1063 return 0;
1064 }
1065 tssa->var_info[ssa_var].type &= info->type;
1066 if (info->ce) {
1067 if (tssa->var_info[ssa_var].ce) {
1068 if (tssa->var_info[ssa_var].ce != info->ce) {
1069 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
1070 /* everything fine */
1071 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
1072 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
1073 } else {
1074 // TODO: classes may implement the same interface ???
1075 //ZEND_UNREACHABLE();
1076 }
1077 }
1078 tssa->var_info[ssa_var].is_instanceof =
1079 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
1080 } else {
1081 tssa->var_info[ssa_var].ce = info->ce;
1082 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
1083 }
1084 }
1085 if (info->has_range) {
1086 if (tssa->var_info[ssa_var].has_range) {
1087 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
1088 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
1089 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
1090 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
1091 } else {
1092 tssa->var_info[ssa_var].has_range = 1;
1093 tssa->var_info[ssa_var].range = info->range;
1094 }
1095 }
1096 return 1;
1097 }
1098 return 0;
1099 }
1100
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)1101 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
1102 {
1103 while (1) {
1104 if (p->op == ZEND_JIT_TRACE_VM) {
1105 if (p->opline->opcode == ZEND_DO_UCALL
1106 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
1107 || p->opline->opcode == ZEND_DO_FCALL) {
1108 if (p->opline->result_type != IS_UNUSED) {
1109 return ssa_op->result_def;
1110 }
1111 }
1112 return -1;
1113 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1114 /*skip */
1115 } else {
1116 return -1;
1117 }
1118 p--;
1119 }
1120 }
1121
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1122 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1123 {
1124 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1125 p--;
1126 while (1) {
1127 if (p->op == ZEND_JIT_TRACE_VM) {
1128 if (p->opline->opcode == ZEND_INIT_FCALL
1129 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1130 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1131 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1132 || p->opline->opcode == ZEND_INIT_USER_CALL
1133 || p->opline->opcode == ZEND_NEW
1134 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1135 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
1136 || p->opline->opcode == ZEND_INIT_PARENT_PROPERTY_HOOK_CALL) {
1137 return p->opline;
1138 }
1139 return NULL;
1140 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1141 /*skip */
1142 } else {
1143 return NULL;
1144 }
1145 p--;
1146 }
1147 } else {
1148 const zend_op *opline = NULL;
1149 int call_level = 0;
1150
1151 p++;
1152 while (1) {
1153 if (p->op == ZEND_JIT_TRACE_VM) {
1154 opline = p->opline;
1155 break;
1156 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1157 call_level++;
1158 /*skip */
1159 } else {
1160 return NULL;
1161 }
1162 p--;
1163 }
1164 if (opline) {
1165 while (opline > op_array->opcodes) {
1166 opline--;
1167 switch (opline->opcode) {
1168 case ZEND_INIT_FCALL:
1169 case ZEND_INIT_FCALL_BY_NAME:
1170 case ZEND_INIT_NS_FCALL_BY_NAME:
1171 case ZEND_INIT_METHOD_CALL:
1172 case ZEND_INIT_DYNAMIC_CALL:
1173 case ZEND_INIT_STATIC_METHOD_CALL:
1174 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
1175 case ZEND_INIT_USER_CALL:
1176 case ZEND_NEW:
1177 if (call_level == 0) {
1178 return opline;
1179 }
1180 call_level--;
1181 break;
1182 case ZEND_DO_FCALL:
1183 case ZEND_DO_ICALL:
1184 case ZEND_DO_UCALL:
1185 case ZEND_DO_FCALL_BY_NAME:
1186 case ZEND_CALLABLE_CONVERT:
1187 call_level++;
1188 break;
1189 }
1190 }
1191 }
1192 }
1193 return NULL;
1194 }
1195
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1196 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1197 {
1198 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1199 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1200 int idx = tssa->vars[var].definition;
1201
1202 if (idx >= 0) {
1203 if (tssa->ops[idx].op1_def == var) {
1204 const zend_op *opline = ssa_opcodes[idx];
1205 if (opline->opcode == ZEND_PRE_DEC
1206 || opline->opcode == ZEND_PRE_INC
1207 || opline->opcode == ZEND_POST_DEC
1208 || opline->opcode == ZEND_POST_INC) {
1209 if (tssa->ops[idx].op1_use >= 0
1210 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1211 return 0;
1212 }
1213 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1214 return 0;
1215 }
1216 return 1;
1217 } else if (opline->opcode == ZEND_ASSIGN_OP
1218 && (opline->extended_value == ZEND_ADD
1219 || opline->extended_value == ZEND_SUB
1220 || opline->extended_value == ZEND_MUL)) {
1221 if ((opline->op2_type & (IS_VAR|IS_CV))
1222 && tssa->ops[idx].op2_use >= 0
1223 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1224 return 0;
1225 }
1226 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1227 return 0;
1228 }
1229 if (opline->op2_type == IS_CONST) {
1230 zval *zv = RT_CONSTANT(opline, opline->op2);
1231 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1232 return 0;
1233 }
1234 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1235 return 0;
1236 }
1237 return 1;
1238 }
1239 }
1240 if (tssa->ops[idx].result_def == var) {
1241 const zend_op *opline = ssa_opcodes[idx];
1242 if (opline->opcode == ZEND_ADD
1243 || opline->opcode == ZEND_SUB
1244 || opline->opcode == ZEND_MUL) {
1245 if ((opline->op1_type & (IS_VAR|IS_CV))
1246 && tssa->ops[idx].op1_use >= 0
1247 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1248 return 0;
1249 }
1250 if ((opline->op2_type & (IS_VAR|IS_CV))
1251 && tssa->ops[idx].op2_use >= 0
1252 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1253 return 0;
1254 }
1255 if (opline->op1_type == IS_CONST) {
1256 zval *zv = RT_CONSTANT(opline, opline->op1);
1257 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1258 return 0;
1259 }
1260 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1261 return 0;
1262 }
1263 if (opline->op2_type == IS_CONST) {
1264 zval *zv = RT_CONSTANT(opline, opline->op2);
1265 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1266 return 0;
1267 }
1268 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1269 return 0;
1270 }
1271 return 1;
1272 } else if (opline->opcode == ZEND_PRE_DEC
1273 || opline->opcode == ZEND_PRE_INC
1274 || opline->opcode == ZEND_POST_DEC
1275 || opline->opcode == ZEND_POST_INC) {
1276 if ((opline->op1_type & (IS_VAR|IS_CV))
1277 && tssa->ops[idx].op1_use >= 0
1278 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1279 return 0;
1280 }
1281 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1282 return 0;
1283 }
1284 return 1;
1285 }
1286 }
1287 }
1288 }
1289 return 0;
1290 }
1291
1292 typedef struct _zend_tssa {
1293 zend_ssa ssa;
1294 const zend_op **tssa_opcodes;
1295 int used_stack;
1296 } zend_tssa;
1297
1298 static const zend_op _nop_opcode = {0};
1299
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1300 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1301 {
1302 zend_ssa *tssa;
1303 zend_ssa_op *ssa_ops, *op;
1304 zend_ssa_var *ssa_vars;
1305 zend_ssa_var_info *ssa_var_info;
1306 const zend_op_array *op_array;
1307 const zend_op *opline;
1308 const zend_op **ssa_opcodes;
1309 zend_jit_trace_rec *p;
1310 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1311 zend_jit_trace_stack *stack;
1312 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1313 uint32_t optimization_level = 0;
1314 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1315 size_t frame_size, stack_top, stack_size, stack_bottom;
1316 zend_jit_op_array_trace_extension *jit_extension;
1317 zend_ssa *ssa;
1318 zend_jit_trace_stack_frame *frame, *top, *call;
1319 zend_ssa_var_info return_value_info;
1320
1321 /* 1. Count number of TSSA opcodes;
1322 * Count number of activation frames;
1323 * Calculate size of abstract stack;
1324 * Construct regular SSA for involved op_array */
1325 op_array = trace_buffer->op_array;
1326 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1327 stack_bottom = 0;
1328 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1329 ssa_ops_count = 0;
1330 call_level = 0;
1331 level = 0;
1332 num_op_arrays = 0;
1333 /* Remember op_array to cleanup */
1334 op_arrays[num_op_arrays++] = op_array;
1335 /* Build SSA */
1336 ssa = zend_jit_trace_build_ssa(op_array, script);
1337 for (;;p++) {
1338 if (p->op == ZEND_JIT_TRACE_VM) {
1339 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1340 const zend_op *opline = p->opline;
1341
1342 switch (opline->opcode) {
1343 case ZEND_INCLUDE_OR_EVAL:
1344 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1345 break;
1346 case ZEND_FETCH_R:
1347 case ZEND_FETCH_W:
1348 case ZEND_FETCH_RW:
1349 case ZEND_FETCH_FUNC_ARG:
1350 case ZEND_FETCH_IS:
1351 case ZEND_FETCH_UNSET:
1352 case ZEND_UNSET_VAR:
1353 case ZEND_ISSET_ISEMPTY_VAR:
1354 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1355 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1356 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1357 !op_array->function_name) {
1358 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1359 }
1360 break;
1361 }
1362 }
1363 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1364 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1365 call_level++;
1366 stack_top += zend_jit_trace_frame_size(p->op_array);
1367 if (stack_top > stack_size) {
1368 stack_size = stack_top;
1369 }
1370 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1371 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1372 if (p->func
1373 && p->func != (zend_function*)&zend_pass_function
1374 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1375 || zend_string_equals_literal(p->func->common.function_name, "compact")
1376 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1377 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1378 }
1379 }
1380 frame_size = zend_jit_trace_frame_size(p->op_array);
1381 if (call_level == 0) {
1382 if (stack_top + frame_size > stack_size) {
1383 stack_size = stack_top + frame_size;
1384 }
1385 } else {
1386 call_level--;
1387 stack_top -= frame_size;
1388 }
1389 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1390 op_array = p->op_array;
1391 if (call_level == 0) {
1392 stack_top += zend_jit_trace_frame_size(op_array);
1393 if (stack_top > stack_size) {
1394 stack_size = stack_top;
1395 }
1396 } else {
1397 call_level--;
1398 }
1399 level++;
1400 jit_extension =
1401 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1402 ssa = &jit_extension->func_info.ssa;
1403 if (ssa->cfg.blocks_count) {
1404 /* pass */
1405 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1406 /* Too many functions in single trace */
1407 *num_op_arrays_ptr = num_op_arrays;
1408 return NULL;
1409 } else {
1410 /* Remember op_array to cleanup */
1411 op_arrays[num_op_arrays++] = op_array;
1412 /* Build SSA */
1413 ssa = zend_jit_trace_build_ssa(op_array, script);
1414 }
1415 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1416 if (level == 0) {
1417 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1418 jit_extension =
1419 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1420 ssa = &jit_extension->func_info.ssa;
1421 if (ssa->cfg.blocks_count) {
1422 /* pass */
1423 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1424 /* Too many functions in single trace */
1425 *num_op_arrays_ptr = num_op_arrays;
1426 return NULL;
1427 } else {
1428 /* Remember op_array to cleanup */
1429 op_arrays[num_op_arrays++] = op_array;
1430 /* Build SSA */
1431 ssa = zend_jit_trace_build_ssa(op_array, script);
1432 }
1433 } else {
1434 stack_top -= zend_jit_trace_frame_size(op_array);
1435 level--;
1436 }
1437 op_array = p->op_array;
1438 } else if (p->op == ZEND_JIT_TRACE_END) {
1439 break;
1440 }
1441 }
1442 *num_op_arrays_ptr = num_op_arrays;
1443
1444 /* Allocate space for abstract stack */
1445 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1446
1447 /* 2. Construct TSSA */
1448 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1449 tssa->cfg.flags = ZEND_SSA_TSSA;
1450 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1451 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1452 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1453
1454 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1455 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1456 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1457 tssa->cfg.blocks_count = 2;
1458 tssa->cfg.edges_count = 2;
1459
1460 tssa->cfg.predecessors[0] = 0;
1461 tssa->cfg.predecessors[1] = 1;
1462
1463 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1464 tssa->cfg.blocks[0].successors_count = 1;
1465 tssa->cfg.blocks[0].predecessors_count = 0;
1466 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1467 tssa->cfg.blocks[0].successors[0] = 1;
1468
1469 tssa->cfg.blocks[1].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1470 tssa->cfg.blocks[1].len = ssa_ops_count;
1471 tssa->cfg.blocks[1].successors_count = 1;
1472 tssa->cfg.blocks[1].predecessors_count = 2;
1473 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1474 tssa->cfg.blocks[1].successors[1] = 1;
1475 } else {
1476 tssa->cfg.blocks_count = 1;
1477 tssa->cfg.edges_count = 0;
1478
1479 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1480 tssa->cfg.blocks[0].len = ssa_ops_count;
1481 tssa->cfg.blocks[0].successors_count = 0;
1482 tssa->cfg.blocks[0].predecessors_count = 0;
1483 }
1484 ((zend_tssa*)tssa)->used_stack = -1;
1485
1486 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1487 return tssa;
1488 }
1489
1490 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1491 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1492 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1493 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1494 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1495
1496 op_array = trace_buffer->op_array;
1497 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1498 ssa_vars_count = op_array->last_var;
1499 } else {
1500 ssa_vars_count = op_array->last_var + op_array->T;
1501 }
1502 stack = frame->stack;
1503 for (i = 0; i < ssa_vars_count; i++) {
1504 SET_STACK_VAR(stack, i, i);
1505 }
1506
1507 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1508 // TODO: For tracing, it's possible, to create pseudo Phi functions
1509 // at the end of loop, without this additional pass (like LuaJIT) ???
1510 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1511 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1512 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1513 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1514 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1515 }
1516
1517 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1518 idx = 0;
1519 level = 0;
1520 for (;;p++) {
1521 if (p->op == ZEND_JIT_TRACE_VM) {
1522 opline = p->opline;
1523 ssa_opcodes[idx] = opline;
1524 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1525 idx++;
1526 len = zend_jit_trace_op_len(p->opline);
1527 while (len > 1) {
1528 opline++;
1529 ssa_opcodes[idx] = opline;
1530 if (opline->opcode != ZEND_OP_DATA) {
1531 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1532 }
1533 idx++;
1534 len--;
1535 }
1536 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1537 frame = zend_jit_trace_call_frame(frame, op_array);
1538 stack = frame->stack;
1539 op_array = p->op_array;
1540 level++;
1541 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1542 return NULL;
1543 }
1544 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1545 for (i = 0; i < op_array->last_var; i++) {
1546 SET_STACK_VAR(stack, i, ssa_vars_count++);
1547 }
1548 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1549 op_array = p->op_array;
1550 frame = zend_jit_trace_ret_frame(frame, op_array);
1551 stack = frame->stack;
1552 if (level == 0) {
1553 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1554 return NULL;
1555 }
1556 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1557 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1558 SET_STACK_VAR(stack, i, ssa_vars_count++);
1559 }
1560 } else {
1561 level--;
1562 }
1563 } else if (p->op == ZEND_JIT_TRACE_END) {
1564 break;
1565 }
1566 }
1567
1568 op_array = trace_buffer->op_array;
1569 tssa->vars_count = ssa_vars_count;
1570 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1571 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1572 vars_count = op_array->last_var;
1573 } else {
1574 vars_count = op_array->last_var + op_array->T;
1575 }
1576 i = 0;
1577 while (i < vars_count) {
1578 ssa_vars[i].var = i;
1579 ssa_vars[i].scc = -1;
1580 ssa_vars[i].definition = -1;
1581 ssa_vars[i].use_chain = -1;
1582 i++;
1583 }
1584 while (i < tssa->vars_count) {
1585 ssa_vars[i].var = -1;
1586 ssa_vars[i].scc = -1;
1587 ssa_vars[i].definition = -1;
1588 ssa_vars[i].use_chain = -1;
1589 i++;
1590 }
1591
1592 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1593 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1594 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1595 /* Update Phi sources */
1596 zend_ssa_phi *phi = tssa->blocks[1].phis;
1597
1598 while (phi) {
1599 phi->sources[1] = STACK_VAR(stack, phi->var);
1600 ssa_vars[phi->ssa_var].var = phi->var;
1601 ssa_vars[phi->ssa_var].definition_phi = phi;
1602 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1603 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1604 phi = phi->next;
1605 }
1606 }
1607
1608 /* 3. Compute use-def chains */
1609 idx = (ssa_ops_count - 1);
1610 op = ssa_ops + idx;
1611 while (idx >= 0) {
1612 opline = ssa_opcodes[idx];
1613 if (op->op1_use >= 0) {
1614 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1615 ssa_vars[op->op1_use].use_chain = idx;
1616 }
1617 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1618 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1619 ssa_vars[op->op2_use].use_chain = idx;
1620 }
1621 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1622 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1623 ssa_vars[op->result_use].use_chain = idx;
1624 }
1625 if (op->op1_def >= 0) {
1626 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1627 ssa_vars[op->op1_def].definition = idx;
1628 }
1629 if (op->op2_def >= 0) {
1630 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1631 ssa_vars[op->op2_def].definition = idx;
1632 }
1633 if (op->result_def >= 0) {
1634 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1635 ssa_vars[op->result_def].definition = idx;
1636 }
1637 op--;
1638 idx--;
1639 }
1640
1641 /* 4. Type inference */
1642 op_array = trace_buffer->op_array;
1643 opline = trace_buffer[1].opline;
1644 jit_extension =
1645 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1646 ssa = &jit_extension->func_info.ssa;
1647
1648 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1649
1650 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1651 i = 0;
1652 while (i < op_array->last_var) {
1653 if (i < op_array->num_args) {
1654 if (ssa->var_info
1655 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i, NULL)) {
1656 /* pass */
1657 } else {
1658 if (ssa->vars) {
1659 ssa_vars[i].no_val = ssa->vars[i].no_val;
1660 ssa_vars[i].alias = ssa->vars[i].alias;
1661 } else {
1662 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1663 }
1664 if (op_array->arg_info && i < trace_buffer[1].opline - op_array->opcodes) {
1665 zend_arg_info *arg_info = &op_array->arg_info[i];
1666 zend_class_entry *ce;
1667 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1668
1669 if (ZEND_ARG_SEND_MODE(arg_info)) {
1670 tmp |= MAY_BE_REF;
1671 }
1672 ssa_var_info[i].type = tmp;
1673 ssa_var_info[i].ce = ce;
1674 ssa_var_info[i].is_instanceof = 1;
1675 } else {
1676 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1677 }
1678 }
1679 } else {
1680 if (ssa->vars) {
1681 ssa_vars[i].no_val = ssa->vars[i].no_val;
1682 ssa_vars[i].alias = ssa->vars[i].alias;
1683 } else {
1684 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1685 }
1686 if (ssa_vars[i].alias == NO_ALIAS) {
1687 ssa_var_info[i].type = MAY_BE_UNDEF;
1688 } else {
1689 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1690 }
1691 }
1692 i++;
1693 }
1694 } else {
1695 int parent_vars_count = 0;
1696 zend_jit_trace_stack *parent_stack = NULL;
1697
1698 i = 0;
1699 if (parent_trace) {
1700 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1701 op_array->last_var + op_array->T);
1702 if (parent_vars_count) {
1703 parent_stack =
1704 zend_jit_traces[parent_trace].stack_map +
1705 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1706 }
1707 }
1708 while (i < op_array->last_var + op_array->T) {
1709 if (!ssa->var_info
1710 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i, opline)) {
1711 if (ssa->vars && i < ssa->vars_count) {
1712 ssa_vars[i].alias = ssa->vars[i].alias;
1713 } else {
1714 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1715 }
1716 if (i < op_array->last_var) {
1717 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1718 } else {
1719 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1720 }
1721 }
1722 if (i < parent_vars_count) {
1723 /* Initialize TSSA variable from parent trace */
1724 uint8_t op_type = STACK_TYPE(parent_stack, i);
1725
1726 if (op_type != IS_UNKNOWN) {
1727 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1728 if (!ssa_var_info[i].type
1729 && op_type == IS_UNDEF
1730 && i >= op_array->last_var) {
1731 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1732 ssa_var_info[i].type |= MAY_BE_UNDEF;
1733 }
1734 }
1735 }
1736 i++;
1737 }
1738 }
1739
1740 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1741 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1742 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1743 /* Propagate initial value through Phi functions */
1744 zend_ssa_phi *phi = tssa->blocks[1].phis;
1745
1746 while (phi) {
1747 if (!ssa->var_info
1748 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var, NULL)) {
1749 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1750 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1751 }
1752 phi = phi->next;
1753 }
1754 }
1755
1756 frame = JIT_G(current_frame);
1757 top = zend_jit_trace_call_frame(frame, op_array);
1758 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1759 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1760 frame->used_stack = 0;
1761 memset(&return_value_info, 0, sizeof(return_value_info));
1762
1763 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1764 max_used_stack = used_stack = 0;
1765 } else {
1766 max_used_stack = used_stack = -1;
1767 }
1768
1769 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1770 idx = 0;
1771 level = 0;
1772 opline = NULL;
1773 for (;;p++) {
1774 if (p->op == ZEND_JIT_TRACE_VM) {
1775 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1776 uint8_t val_type = IS_UNKNOWN;
1777 // zend_class_entry *op1_ce = NULL;
1778 zend_class_entry *op2_ce = NULL;
1779
1780 opline = p->opline;
1781
1782 op1_type = orig_op1_type = p->op1_type;
1783 op2_type = orig_op2_type = p->op2_type;
1784 op3_type = p->op3_type;
1785 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1786 op1_type = IS_UNKNOWN;
1787 }
1788 if (op1_type != IS_UNKNOWN) {
1789 op1_type &= ~IS_TRACE_PACKED;
1790 }
1791 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1792 op2_type = IS_UNKNOWN;
1793 }
1794 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1795 op3_type = IS_UNKNOWN;
1796 }
1797
1798 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1799 // op1_ce = (zend_class_entry*)(p+1)->ce;
1800 p++;
1801 }
1802 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1803 op2_ce = (zend_class_entry*)(p+1)->ce;
1804 p++;
1805 }
1806 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1807 val_type = (p+1)->op1_type;
1808 p++;
1809 }
1810
1811 switch (opline->opcode) {
1812 case ZEND_ASSIGN_OP:
1813 if (opline->extended_value == ZEND_POW
1814 || opline->extended_value == ZEND_DIV) {
1815 // TODO: check for division by zero ???
1816 break;
1817 }
1818 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1819 break;
1820 }
1821 ADD_OP1_TRACE_GUARD();
1822 ADD_OP2_TRACE_GUARD();
1823 break;
1824 case ZEND_ASSIGN_DIM_OP:
1825 if (opline->extended_value == ZEND_POW
1826 || opline->extended_value == ZEND_DIV) {
1827 // TODO: check for division by zero ???
1828 break;
1829 }
1830 if (opline->result_type != IS_UNUSED) {
1831 break;
1832 }
1833 if (op3_type != IS_UNKNOWN
1834 && !zend_jit_supported_binary_op(
1835 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1836 break;
1837 }
1838 ZEND_FALLTHROUGH;
1839 case ZEND_ASSIGN_DIM:
1840 if (opline->op1_type == IS_CV) {
1841 if ((opline+1)->op1_type == IS_CV
1842 && (opline+1)->op1.var == opline->op1.var) {
1843 /* skip $a[x] = $a; */
1844 break;
1845 }
1846 ADD_OP1_DATA_TRACE_GUARD();
1847 ADD_OP2_TRACE_GUARD();
1848 ADD_OP1_TRACE_GUARD();
1849 } else if (orig_op1_type != IS_UNKNOWN
1850 && (orig_op1_type & IS_TRACE_INDIRECT)
1851 && opline->result_type == IS_UNUSED) {
1852 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1853 ADD_OP1_DATA_TRACE_GUARD();
1854 }
1855 ADD_OP2_TRACE_GUARD();
1856 }
1857 if (op1_type == IS_ARRAY
1858 && ((opline->op2_type == IS_CONST
1859 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1860 || (opline->op2_type != IS_CONST
1861 && op2_type == IS_LONG))) {
1862
1863 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1864 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1865
1866 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1867 info->type |= MAY_BE_PACKED_GUARD;
1868 info->type &= ~MAY_BE_ARRAY_PACKED;
1869 }
1870 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1871 && val_type != IS_UNKNOWN
1872 && val_type != IS_UNDEF) {
1873 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1874
1875 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1876 info->type |= MAY_BE_PACKED_GUARD;
1877 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1878 }
1879 }
1880 }
1881 break;
1882 case ZEND_ASSIGN_OBJ_OP:
1883 if (opline->extended_value == ZEND_POW
1884 || opline->extended_value == ZEND_DIV) {
1885 // TODO: check for division by zero ???
1886 break;
1887 }
1888 if (opline->result_type != IS_UNUSED) {
1889 break;
1890 }
1891 ZEND_FALLTHROUGH;
1892 case ZEND_ASSIGN_OBJ:
1893 case ZEND_PRE_INC_OBJ:
1894 case ZEND_PRE_DEC_OBJ:
1895 case ZEND_POST_INC_OBJ:
1896 case ZEND_POST_DEC_OBJ:
1897 if (opline->op2_type != IS_CONST
1898 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1899 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1900 break;
1901 }
1902 if (opline->opcode == ZEND_ASSIGN_OBJ
1903 || opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1904 if (opline->op1_type == IS_CV
1905 && (opline+1)->op1_type == IS_CV
1906 && (opline+1)->op1.var == opline->op1.var) {
1907 /* skip $a->prop += $a; */
1908 break;
1909 }
1910 ADD_OP1_DATA_TRACE_GUARD();
1911 }
1912 ADD_OP1_TRACE_GUARD();
1913 break;
1914 case ZEND_CONCAT:
1915 case ZEND_FAST_CONCAT:
1916 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1917 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1918 ADD_OP2_TRACE_GUARD();
1919 ADD_OP1_TRACE_GUARD();
1920 }
1921 break;
1922 case ZEND_ADD:
1923 case ZEND_SUB:
1924 case ZEND_MUL:
1925 // case ZEND_DIV: // TODO: check for division by zero ???
1926 if (orig_op1_type == IS_UNDEF || orig_op2_type == IS_UNDEF) {
1927 break;
1928 }
1929 ZEND_FALLTHROUGH;
1930 case ZEND_IS_EQUAL:
1931 case ZEND_IS_NOT_EQUAL:
1932 case ZEND_IS_SMALLER:
1933 case ZEND_IS_SMALLER_OR_EQUAL:
1934 case ZEND_CASE:
1935 case ZEND_IS_IDENTICAL:
1936 case ZEND_IS_NOT_IDENTICAL:
1937 case ZEND_CASE_STRICT:
1938 case ZEND_BW_OR:
1939 case ZEND_BW_AND:
1940 case ZEND_BW_XOR:
1941 case ZEND_SL:
1942 case ZEND_SR:
1943 case ZEND_MOD:
1944 ADD_OP2_TRACE_GUARD();
1945 ZEND_FALLTHROUGH;
1946 case ZEND_ECHO:
1947 case ZEND_STRLEN:
1948 case ZEND_COUNT:
1949 case ZEND_QM_ASSIGN:
1950 case ZEND_FE_RESET_R:
1951 ADD_OP1_TRACE_GUARD();
1952 break;
1953 case ZEND_FE_FETCH_R:
1954 ADD_OP1_TRACE_GUARD();
1955 if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) {
1956
1957 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1958
1959 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1960 info->type |= MAY_BE_PACKED_GUARD;
1961 if (orig_op1_type & IS_TRACE_PACKED) {
1962 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1963 } else {
1964 info->type &= ~MAY_BE_ARRAY_PACKED;
1965 }
1966 }
1967 }
1968 break;
1969 case ZEND_VERIFY_RETURN_TYPE:
1970 if (opline->op1_type == IS_UNUSED) {
1971 /* Always throws */
1972 break;
1973 }
1974 if (opline->op1_type == IS_CONST) {
1975 /* TODO Different instruction format, has return value */
1976 break;
1977 }
1978 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1979 /* Not worth bothering with */
1980 break;
1981 }
1982 ADD_OP1_TRACE_GUARD();
1983 break;
1984 case ZEND_FETCH_DIM_FUNC_ARG:
1985 if (!frame
1986 || !frame->call
1987 || !frame->call->func
1988 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1989 break;
1990 }
1991 ADD_OP2_TRACE_GUARD();
1992 ADD_OP1_TRACE_GUARD();
1993 break;
1994 case ZEND_PRE_INC:
1995 case ZEND_PRE_DEC:
1996 case ZEND_POST_INC:
1997 case ZEND_POST_DEC:
1998 if (opline->op1_type != IS_CV) {
1999 break;
2000 }
2001 ADD_OP1_TRACE_GUARD();
2002 break;
2003 case ZEND_ASSIGN:
2004 if (opline->op1_type != IS_CV) {
2005 break;
2006 }
2007 ADD_OP2_TRACE_GUARD();
2008 if (op1_type != IS_UNKNOWN
2009 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
2010 ADD_OP1_TRACE_GUARD();
2011 }
2012 break;
2013 case ZEND_CAST:
2014 if (opline->extended_value != op1_type) {
2015 break;
2016 }
2017 ADD_OP1_TRACE_GUARD();
2018 break;
2019 case ZEND_JMPZ:
2020 case ZEND_JMPNZ:
2021 case ZEND_JMPZ_EX:
2022 case ZEND_JMPNZ_EX:
2023 case ZEND_BOOL:
2024 case ZEND_BOOL_NOT:
2025 ADD_OP1_TRACE_GUARD();
2026 break;
2027 case ZEND_ISSET_ISEMPTY_CV:
2028 if ((opline->extended_value & ZEND_ISEMPTY)) {
2029 // TODO: support for empty() ???
2030 break;
2031 }
2032 ADD_OP1_TRACE_GUARD();
2033 break;
2034 case ZEND_IN_ARRAY:
2035 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
2036 break;
2037 }
2038 ADD_OP1_TRACE_GUARD();
2039 break;
2040 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
2041 if ((opline->extended_value & ZEND_ISEMPTY)) {
2042 // TODO: support for empty() ???
2043 break;
2044 }
2045 ZEND_FALLTHROUGH;
2046 case ZEND_FETCH_DIM_R:
2047 case ZEND_FETCH_DIM_IS:
2048 case ZEND_FETCH_LIST_R:
2049 ADD_OP1_TRACE_GUARD();
2050 ADD_OP2_TRACE_GUARD();
2051
2052 if (op1_type == IS_ARRAY
2053 && opline->op1_type != IS_CONST
2054 && ((opline->op2_type == IS_CONST
2055 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2056 || (opline->op2_type != IS_CONST
2057 && op2_type == IS_LONG))) {
2058
2059 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
2060
2061 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
2062 info->type |= MAY_BE_PACKED_GUARD;
2063 if (orig_op1_type & IS_TRACE_PACKED) {
2064 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2065 } else {
2066 info->type &= ~MAY_BE_ARRAY_PACKED;
2067 }
2068 }
2069 }
2070 break;
2071 case ZEND_FETCH_DIM_W:
2072 case ZEND_FETCH_DIM_RW:
2073 // case ZEND_FETCH_DIM_UNSET:
2074 case ZEND_FETCH_LIST_W:
2075 if (opline->op1_type != IS_CV
2076 && (orig_op1_type == IS_UNKNOWN
2077 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
2078 break;
2079 }
2080 ADD_OP1_TRACE_GUARD();
2081 ADD_OP2_TRACE_GUARD();
2082 if (op1_type == IS_ARRAY
2083 && !(orig_op1_type & IS_TRACE_PACKED)
2084 && ((opline->op2_type == IS_CONST
2085 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2086 || (opline->op2_type != IS_CONST
2087 && op2_type == IS_LONG))) {
2088
2089 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
2090
2091 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
2092 info->type |= MAY_BE_PACKED_GUARD;
2093 info->type &= ~MAY_BE_ARRAY_PACKED;
2094 }
2095 }
2096 break;
2097 case ZEND_SEND_VAL_EX:
2098 case ZEND_SEND_VAR_EX:
2099 case ZEND_SEND_VAR_NO_REF_EX:
2100 if (opline->op2_type == IS_CONST) {
2101 /* Named parameters not supported in JIT */
2102 break;
2103 }
2104 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
2105 goto propagate_arg;
2106 }
2107 ZEND_FALLTHROUGH;
2108 case ZEND_SEND_VAL:
2109 case ZEND_SEND_VAR:
2110 case ZEND_SEND_VAR_NO_REF:
2111 case ZEND_SEND_FUNC_ARG:
2112 if (opline->op2_type == IS_CONST) {
2113 /* Named parameters not supported in JIT */
2114 break;
2115 }
2116 ADD_OP1_TRACE_GUARD();
2117 propagate_arg:
2118 /* Propagate argument type */
2119 if (frame->call
2120 && frame->call->func
2121 && frame->call->func->type == ZEND_USER_FUNCTION
2122 && opline->op2.num <= frame->call->func->op_array.num_args) {
2123 uint32_t info;
2124
2125 if (opline->op1_type == IS_CONST) {
2126 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
2127 } else {
2128 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2129 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
2130 }
2131 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
2132 zend_arg_info *arg_info;
2133
2134 ZEND_ASSERT(frame->call->func->op_array.arg_info);
2135 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
2136 if (ZEND_TYPE_IS_SET(arg_info->type)) {
2137 zend_class_entry *ce;
2138 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2139 info &= tmp;
2140 if (!info) {
2141 break;
2142 }
2143 }
2144 }
2145 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2146 info |= MAY_BE_RCN;
2147 }
2148 if (info & MAY_BE_UNDEF) {
2149 info |= MAY_BE_NULL;
2150 info &= ~MAY_BE_UNDEF;
2151 }
2152 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2153 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2154 }
2155 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2156 }
2157 break;
2158 case ZEND_RETURN:
2159 ADD_OP1_TRACE_GUARD();
2160 /* Propagate return value types */
2161 if (opline->op1_type == IS_UNUSED) {
2162 return_value_info.type = MAY_BE_NULL;
2163 } else if (opline->op1_type == IS_CONST) {
2164 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2165 } else {
2166 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2167 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2168 if (return_value_info.type & MAY_BE_UNDEF) {
2169 return_value_info.type &= ~MAY_BE_UNDEF;
2170 return_value_info.type |= MAY_BE_NULL;
2171 }
2172 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2173 /* CVs are going to be destructed and the reference-counter
2174 of return value may be decremented to 1 */
2175 return_value_info.type |= MAY_BE_RC1;
2176 }
2177 return_value_info.type &= ~MAY_BE_GUARD;
2178 }
2179 break;
2180 case ZEND_CHECK_FUNC_ARG:
2181 if (!frame
2182 || !frame->call
2183 || !frame->call->func) {
2184 break;
2185 }
2186 if (opline->op2_type == IS_CONST
2187 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2188 /* Named parameters not supported in JIT */
2189 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2190 break;
2191 }
2192 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2193 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2194 } else {
2195 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2196 }
2197 break;
2198 case ZEND_FETCH_OBJ_FUNC_ARG:
2199 if (!frame
2200 || !frame->call
2201 || !frame->call->func
2202 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2203 break;
2204 }
2205 ZEND_FALLTHROUGH;
2206 case ZEND_FETCH_OBJ_R:
2207 case ZEND_FETCH_OBJ_IS:
2208 case ZEND_FETCH_OBJ_W:
2209 if (opline->op2_type != IS_CONST
2210 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2211 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2212 break;
2213 }
2214 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2215 ADD_OP1_TRACE_GUARD();
2216 }
2217 break;
2218 case ZEND_INIT_METHOD_CALL:
2219 if (opline->op2_type != IS_CONST
2220 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2221 break;
2222 }
2223 ADD_OP1_TRACE_GUARD();
2224 break;
2225 case ZEND_INIT_DYNAMIC_CALL:
2226 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2227 ADD_OP2_TRACE_GUARD();
2228 }
2229 break;
2230 case ZEND_SEND_ARRAY:
2231 case ZEND_SEND_UNPACK:
2232 case ZEND_CHECK_UNDEF_ARGS:
2233 case ZEND_INCLUDE_OR_EVAL:
2234 max_used_stack = used_stack = -1;
2235 break;
2236 case ZEND_TYPE_CHECK:
2237 if (opline->extended_value == MAY_BE_RESOURCE) {
2238 // TODO: support for is_resource() ???
2239 break;
2240 }
2241 if (op1_type != IS_UNKNOWN
2242 && (opline->extended_value == (1 << op1_type)
2243 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2244 /* add guards only for exact checks, to avoid code duplication */
2245 ADD_OP1_TRACE_GUARD();
2246 }
2247 break;
2248 case ZEND_ROPE_INIT:
2249 case ZEND_ROPE_ADD:
2250 case ZEND_ROPE_END:
2251 if (op2_type == IS_STRING) {
2252 ADD_OP2_TRACE_GUARD();
2253 }
2254 break;
2255 default:
2256 break;
2257 }
2258 len = zend_jit_trace_op_len(opline);
2259 if (ssa->var_info) {
2260 /* Add statically inferred ranges */
2261 if (ssa_ops[idx].op1_def >= 0) {
2262 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2263 }
2264 if (ssa_ops[idx].op2_def >= 0) {
2265 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2266 }
2267 if (ssa_ops[idx].result_def >= 0) {
2268 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2269 }
2270 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2271 if (ssa_ops[idx+1].op1_def >= 0) {
2272 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2273 }
2274 if (ssa_ops[idx+1].op2_def >= 0) {
2275 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2276 }
2277 if (ssa_ops[idx+1].result_def >= 0) {
2278 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2279 }
2280 }
2281 } else {
2282 if (ssa_ops[idx].op1_def >= 0) {
2283 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2284 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2285 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2286 }
2287 }
2288 if (ssa_ops[idx].op2_def >= 0) {
2289 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2290 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2291 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2292 }
2293 }
2294 if (ssa_ops[idx].result_def >= 0) {
2295 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2296 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2297 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2298 }
2299 }
2300 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2301 if (ssa_ops[idx+1].op1_def >= 0) {
2302 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2303 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2304 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2305 }
2306 }
2307 if (ssa_ops[idx+1].op2_def >= 0) {
2308 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2309 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2310 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2311 }
2312 }
2313 if (ssa_ops[idx+1].result_def >= 0) {
2314 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2315 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2316 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2317 }
2318 }
2319 }
2320 }
2321 if (opline->opcode == ZEND_RECV_INIT
2322 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2323 /* RECV_INIT always copy the constant */
2324 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2325 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2326 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2327 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2328 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2329 }
2330 } else {
2331 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2332 // TODO:
2333 assert(0);
2334 }
2335 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2336 && ssa_ops[idx].op1_def >= 0
2337 && op1_type == IS_ARRAY
2338 && (orig_op1_type & IS_TRACE_PACKED)
2339 && val_type != IS_UNKNOWN
2340 && val_type != IS_UNDEF
2341 && ((opline->op2_type == IS_CONST
2342 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2343 || (opline->op2_type != IS_CONST
2344 && op2_type == IS_LONG))) {
2345 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2346
2347 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2348 }
2349 }
2350 if (ssa->var_info) {
2351 /* Add statically inferred restrictions */
2352 if (ssa_ops[idx].op1_def >= 0) {
2353 if (opline->opcode == ZEND_SEND_VAR_EX
2354 && frame
2355 && frame->call
2356 && frame->call->func
2357 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2358 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2359 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2360 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2361 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2362 }
2363 } else {
2364 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2365 }
2366 }
2367 if (ssa_ops[idx].op2_def >= 0) {
2368 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2369 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2370 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2371 }
2372 }
2373 if (ssa_ops[idx].result_def >= 0) {
2374 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2375 }
2376 }
2377 idx++;
2378 while (len > 1) {
2379 opline++;
2380 if (opline->opcode != ZEND_OP_DATA) {
2381 if (ssa->var_info) {
2382 /* Add statically inferred ranges */
2383 if (ssa_ops[idx].op1_def >= 0) {
2384 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2385 }
2386 if (ssa_ops[idx].op2_def >= 0) {
2387 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2388 }
2389 if (ssa_ops[idx].result_def >= 0) {
2390 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2391 }
2392 } else {
2393 if (ssa_ops[idx].op1_def >= 0) {
2394 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2395 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2396 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2397 }
2398 }
2399 if (ssa_ops[idx].op2_def >= 0) {
2400 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2401 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2402 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2403 }
2404 }
2405 if (ssa_ops[idx].result_def >= 0) {
2406 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2407 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2408 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2409 }
2410 }
2411 }
2412 if (opline->opcode == ZEND_RECV_INIT
2413 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2414 /* RECV_INIT always copy the constant */
2415 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2416 } else {
2417 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2418 // TODO:
2419 assert(0);
2420 }
2421 }
2422 }
2423 if (ssa->var_info) {
2424 /* Add statically inferred restrictions */
2425 if (ssa_ops[idx].op1_def >= 0) {
2426 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2427 }
2428 if (ssa_ops[idx].op2_def >= 0) {
2429 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2430 }
2431 if (ssa_ops[idx].result_def >= 0) {
2432 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2433 }
2434 }
2435 idx++;
2436 len--;
2437 }
2438
2439 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2440 op_array = p->op_array;
2441 jit_extension =
2442 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2443 ssa = &jit_extension->func_info.ssa;
2444
2445 call = frame->call;
2446 if (!call) {
2447 /* Trace missed INIT_FCALL opcode */
2448 call = top;
2449 TRACE_FRAME_INIT(call, op_array, 0, 0);
2450 call->used_stack = 0;
2451 top = zend_jit_trace_call_frame(top, op_array);
2452 } else {
2453 ZEND_ASSERT(&call->func->op_array == op_array);
2454 }
2455 frame->call = call->prev;
2456 call->prev = frame;
2457 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2458 frame = call;
2459
2460 level++;
2461 i = 0;
2462 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2463 while (i < op_array->last_var) {
2464 ssa_vars[v].var = i;
2465 if (i < op_array->num_args) {
2466 if (ssa->var_info
2467 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2468 /* pass */
2469 } else {
2470 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2471 if (op_array->arg_info) {
2472 zend_arg_info *arg_info = &op_array->arg_info[i];
2473 zend_class_entry *ce;
2474 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2475
2476 if (ZEND_ARG_SEND_MODE(arg_info)) {
2477 tmp |= MAY_BE_REF;
2478 }
2479 ssa_var_info[v].type = tmp;
2480 ssa_var_info[v].ce = ce;
2481 ssa_var_info[v].is_instanceof = 1;
2482 } else {
2483 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2484 }
2485 }
2486 } else {
2487 if (ssa->vars) {
2488 ssa_vars[v].no_val = ssa->vars[i].no_val;
2489 ssa_vars[v].alias = ssa->vars[i].alias;
2490 } else {
2491 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2492 }
2493 if (ssa_vars[v].alias == NO_ALIAS) {
2494 ssa_var_info[v].type = MAY_BE_UNDEF;
2495 } else {
2496 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2497 }
2498 }
2499 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2500 && i < op_array->num_args) {
2501 /* Propagate argument type */
2502 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2503 }
2504 i++;
2505 v++;
2506 }
2507 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2508 op_array = p->op_array;
2509 jit_extension =
2510 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2511 ssa = &jit_extension->func_info.ssa;
2512 if (level == 0) {
2513 i = 0;
2514 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2515 while (i < op_array->last_var) {
2516 ssa_vars[v].var = i;
2517 if (!ssa->var_info
2518 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2519 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2520 }
2521 i++;
2522 v++;
2523 }
2524 while (i < op_array->last_var + op_array->T) {
2525 ssa_vars[v].var = i;
2526 if (!ssa->var_info
2527 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2528 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2529 }
2530 i++;
2531 v++;
2532 }
2533 if (return_value_info.type != 0) {
2534 zend_jit_trace_rec *q = p + 1;
2535 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2536 q++;
2537 }
2538 if (q->op == ZEND_JIT_TRACE_VM
2539 || (q->op == ZEND_JIT_TRACE_END
2540 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2541 const zend_op *opline = q->opline - 1;
2542 if (opline->result_type != IS_UNUSED) {
2543 ssa_var_info[
2544 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2545 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2546 }
2547 }
2548 memset(&return_value_info, 0, sizeof(return_value_info));
2549 }
2550 } else {
2551 level--;
2552 if (return_value_info.type != 0) {
2553 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2554 const zend_op *opline = (p+1)->opline - 1;
2555 if (opline->result_type != IS_UNUSED) {
2556 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2557 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2558 }
2559 }
2560 }
2561 memset(&return_value_info, 0, sizeof(return_value_info));
2562 }
2563 }
2564
2565 top = frame;
2566 if (frame->prev) {
2567 if (used_stack > 0) {
2568 used_stack -= frame->used_stack;
2569 }
2570 frame = frame->prev;
2571 ZEND_ASSERT(&frame->func->op_array == op_array);
2572 } else {
2573 max_used_stack = used_stack = -1;
2574 frame = zend_jit_trace_ret_frame(frame, op_array);
2575 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2576 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2577 frame->used_stack = 0;
2578 }
2579
2580 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2581 call = top;
2582 TRACE_FRAME_INIT(call, p->func, 0, 0);
2583 call->prev = frame->call;
2584 call->used_stack = 0;
2585 frame->call = call;
2586 top = zend_jit_trace_call_frame(top, p->op_array);
2587 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2588 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2589 SET_STACK_INFO(call->stack, i, -1);
2590 }
2591 }
2592 if (used_stack >= 0
2593 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2594 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2595 max_used_stack = used_stack = -1;
2596 } else {
2597 const zend_op *opline = (p-1)->opline;
2598
2599 switch (opline->opcode) {
2600 case ZEND_INIT_FCALL:
2601 case ZEND_INIT_FCALL_BY_NAME:
2602 case ZEND_INIT_NS_FCALL_BY_NAME:
2603 case ZEND_INIT_METHOD_CALL:
2604 case ZEND_INIT_DYNAMIC_CALL:
2605 //case ZEND_INIT_STATIC_METHOD_CALL:
2606 //case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
2607 //case ZEND_INIT_USER_CALL:
2608 //case ZEND_NEW:
2609 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2610 used_stack += frame->used_stack;
2611 if (used_stack > max_used_stack) {
2612 max_used_stack = used_stack;
2613 }
2614 break;
2615 default:
2616 max_used_stack = used_stack = -1;
2617 }
2618 }
2619 }
2620 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2621 call = frame->call;
2622 if (call) {
2623 top = call;
2624 frame->call = call->prev;
2625 }
2626
2627 if (idx > 0
2628 && ssa_ops[idx-1].result_def >= 0
2629 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2630 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2631 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2632 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2633 opline->opcode == ZEND_DO_FCALL ||
2634 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2635
2636 if (opline->result_type != IS_UNDEF) {
2637 zend_class_entry *ce;
2638 const zend_function *func = p->func;
2639 zend_arg_info *ret_info = func->common.arg_info - 1;
2640 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2641
2642 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2643 }
2644 }
2645 } else if (p->op == ZEND_JIT_TRACE_END) {
2646 break;
2647 }
2648 }
2649
2650 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2651
2652 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2653 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2654 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2655 /* Propagate guards through Phi sources */
2656 zend_ssa_phi *phi = tssa->blocks[1].phis;
2657
2658 op_array = trace_buffer->op_array;
2659 jit_extension =
2660 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2661 ssa = &jit_extension->func_info.ssa;
2662
2663 while (phi) {
2664 uint32_t t = ssa_var_info[phi->ssa_var].type;
2665
2666 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2667 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2668 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2669
2670 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2671 if (!((t0 | t1) & MAY_BE_GUARD)) {
2672 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2673 }
2674 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2675 if (!(t1 & MAY_BE_GUARD)
2676 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2677 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2678 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2679 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2680 MAY_BE_GUARD;
2681 if (!(t0 & MAY_BE_ARRAY)) {
2682 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2683 }
2684 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2685 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2686 }
2687 ssa_var_info[phi->sources[0]].type = t0;
2688 ssa_var_info[phi->sources[0]].type = t0;
2689 }
2690 } else {
2691 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2692 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2693 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2694 MAY_BE_GUARD;
2695 if (!(t0 & MAY_BE_ARRAY)) {
2696 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2697 }
2698 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2699 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2700 }
2701 ssa_var_info[phi->sources[0]].type = t0;
2702 }
2703 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2704 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2705 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2706 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2707 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2708 MAY_BE_GUARD;
2709 if (!(t1 & MAY_BE_ARRAY)) {
2710 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2711 }
2712 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2713 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2714 }
2715 ssa_var_info[phi->sources[1]].type = t1;
2716 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2717 }
2718 }
2719 }
2720 t = ssa_var_info[phi->ssa_var].type;
2721 }
2722
2723 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2724 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2725 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2726
2727 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2728 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2729 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2730 }
2731 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2732 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2733 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2734 ssa_var_info[phi->sources[0]].type =
2735 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2736 }
2737 }
2738 }
2739 phi = phi->next;
2740 }
2741 }
2742
2743 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2744 if (parent_trace) {
2745 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2746 ZEND_JIT_TRACE_NUM,
2747 parent_trace,
2748 exit_num,
2749 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2750 trace_buffer->op_array->scope ? "::" : "",
2751 trace_buffer->op_array->function_name ?
2752 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2753 ZSTR_VAL(trace_buffer->op_array->filename),
2754 trace_buffer[1].opline->lineno);
2755 } else {
2756 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2757 ZEND_JIT_TRACE_NUM,
2758 zend_jit_trace_star_desc(trace_buffer->start),
2759 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2760 trace_buffer->op_array->scope ? "::" : "",
2761 trace_buffer->op_array->function_name ?
2762 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2763 ZSTR_VAL(trace_buffer->op_array->filename),
2764 trace_buffer[1].opline->lineno);
2765 }
2766 zend_jit_dump_trace(trace_buffer, tssa);
2767 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2768 uint32_t idx = trace_buffer[1].last;
2769 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2770 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2771 ZEND_JIT_TRACE_NUM,
2772 link_to);
2773 } else {
2774 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2775 ZEND_JIT_TRACE_NUM,
2776 zend_jit_trace_stop_description[trace_buffer->stop]);
2777 }
2778 }
2779
2780 return tssa;
2781 }
2782
2783 #define RA_HAS_IVAL(var) (ra[var].ref != 0)
2784 #define RA_IVAL_FLAGS(var) ra[var].flags
2785 #define RA_IVAL_START(var, line) do {ra[var].ref = IR_NULL;} while (0)
2786 #define RA_IVAL_END(var, line)
2787 #define RA_IVAL_CLOSE(var, line)
2788 #define RA_IVAL_DEL(var) do {ra[var].ref = IR_UNUSED;} while (0)
2789 #define RA_HAS_REG(var) (ra[var].ref != 0)
2790 #define RA_REG_FLAGS(var) ra[var].flags
2791 #define RA_REG_START(var, line) do {ra[var].ref = IR_NULL;} while (0)
2792 #define RA_REG_DEL(var) do {ra[var].ref = IR_UNUSED;} while (0)
2793
zend_jit_trace_use_var(int line,int var,int def,int use_chain,zend_jit_reg_var * ra,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2794 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, zend_jit_reg_var *ra, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2795 {
2796 ZEND_ASSERT(RA_HAS_IVAL(var));
2797 ZEND_ASSERT(!(RA_IVAL_FLAGS(var) & ZREG_LAST_USE));
2798 RA_IVAL_END(var, line);
2799 if (def >= 0) {
2800 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2801 } else if (use_chain < 0 && (RA_IVAL_FLAGS(var) & (ZREG_LOAD|ZREG_STORE))) {
2802 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2803 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2804 /* pass */
2805 } else if (op_array_ssa->vars) {
2806 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2807
2808 if (ssa->ops[line].op1_use == var) {
2809 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2810 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2811 }
2812 } else if (ssa->ops[line].op2_use == var) {
2813 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2814 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2815 }
2816 } else if (ssa->ops[line].result_use == var) {
2817 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2818 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2819 }
2820 }
2821 }
2822 }
2823
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2824 static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2825 {
2826 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2827 zend_jit_trace_rec *p;
2828 const zend_op_array *op_array;
2829 zend_jit_op_array_trace_extension *jit_extension;
2830 const zend_ssa *op_array_ssa;
2831 const zend_ssa_op *ssa_op;
2832 int i, j, idx, count, level;
2833 zend_jit_reg_var *ra;
2834 const zend_op_array **vars_op_array;
2835 void *checkpoint;
2836 zend_jit_trace_stack_frame *frame;
2837 zend_jit_trace_stack *stack;
2838 uint32_t parent_vars_count = parent_trace ?
2839 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2840 zend_jit_trace_stack *parent_stack = parent_vars_count ?
2841 zend_jit_traces[parent_trace].stack_map +
2842 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2843 checkpoint = zend_arena_checkpoint(CG(arena));
2844 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
2845 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2846 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2847
2848 op_array = trace_buffer->op_array;
2849 jit_extension =
2850 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2851 op_array_ssa = &jit_extension->func_info.ssa;
2852 frame = JIT_G(current_frame);
2853 frame->prev = NULL;
2854 frame->func = (const zend_function*)op_array;
2855 stack = frame->stack;
2856
2857 count = 0;
2858
2859 i = 0;
2860 j = op_array->last_var;
2861 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2862 j += op_array->T;
2863 }
2864 while (i < j) {
2865 SET_STACK_VAR(stack, i, i);
2866 vars_op_array[i] = op_array;
2867 /* We don't start intervals for variables used in Phi */
2868 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2869 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2870 && ssa->vars[i].alias == NO_ALIAS
2871 && zend_jit_var_supports_reg(ssa, i)) {
2872 RA_IVAL_START(i, 0);
2873 if (i < parent_vars_count
2874 && STACK_REG(parent_stack, i) != ZREG_NONE
2875 && STACK_FLAGS(parent_stack, i) != ZREG_ZVAL_COPY
2876 ) {
2877 /* We will try to reuse register from parent trace */
2878 RA_IVAL_FLAGS(i) = STACK_FLAGS(parent_stack, i);
2879 count += 2;
2880 } else {
2881 RA_IVAL_FLAGS(i) = ZREG_LOAD;
2882 count++;
2883 }
2884 }
2885 i++;
2886 }
2887
2888 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2889 j = op_array->last_var + op_array->T;
2890 while (i < j) {
2891 SET_STACK_VAR(stack, i, -1);
2892 i++;
2893 }
2894 }
2895
2896 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2897 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2898 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2899 zend_ssa_phi *phi = ssa->blocks[1].phis;
2900
2901 while (phi) {
2902 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2903 vars_op_array[phi->ssa_var] = op_array;
2904 if (ssa->vars[phi->ssa_var].use_chain >= 0
2905 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2906 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2907 RA_IVAL_START(phi->ssa_var, 0);
2908 count++;
2909 }
2910 phi = phi->next;
2911 }
2912 }
2913
2914 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2915 level = 0;
2916 ssa_op = ssa->ops;
2917 idx = 0;
2918 for (;;p++) {
2919 if (p->op == ZEND_JIT_TRACE_VM) {
2920 const zend_op *opline = p->opline;
2921 int len;
2922 bool support_opline;
2923
2924 support_opline =
2925 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2926
2927 if (support_opline
2928 && opline->opcode == ZEND_ASSIGN
2929 && opline->op1_type == IS_CV
2930 && ssa_op->op1_def >= 0
2931 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2932 /* avoid register allocation in case of possibility of indirect modification*/
2933 support_opline = 0;
2934 }
2935
2936 if (ssa_op->op1_use >= 0
2937 && RA_HAS_IVAL(ssa_op->op1_use)) {
2938 if (!support_opline) {
2939 RA_IVAL_DEL(ssa_op->op1_use);
2940 count--;
2941 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2942 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain,
2943 ra,
2944 ssa, ssa_opcodes, op_array, op_array_ssa);
2945 if (opline->op1_type != IS_CV) {
2946 if (opline->opcode == ZEND_CASE
2947 || opline->opcode == ZEND_CASE_STRICT
2948 || opline->opcode == ZEND_SWITCH_LONG
2949 || opline->opcode == ZEND_MATCH
2950 || opline->opcode == ZEND_FETCH_LIST_R
2951 || opline->opcode == ZEND_COPY_TMP
2952 || opline->opcode == ZEND_SWITCH_STRING
2953 || opline->opcode == ZEND_FE_FETCH_R
2954 || opline->opcode == ZEND_FE_FETCH_RW
2955 || opline->opcode == ZEND_FETCH_LIST_W
2956 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2957 || opline->opcode == ZEND_BIND_LEXICAL
2958 || opline->opcode == ZEND_ROPE_ADD) {
2959 /* The value is kept alive and may be used outside of the trace */
2960 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_STORE;
2961 } else {
2962 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_LAST_USE;
2963 }
2964 }
2965 }
2966 }
2967 if (ssa_op->op2_use >= 0
2968 && ssa_op->op2_use != ssa_op->op1_use
2969 && RA_HAS_IVAL(ssa_op->op2_use)) {
2970 if (!support_opline) {
2971 RA_IVAL_DEL(ssa_op->op2_use);
2972 count--;
2973 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2974 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain,
2975 ra,
2976 ssa, ssa_opcodes, op_array, op_array_ssa);
2977 if (opline->op2_type != IS_CV) {
2978 RA_IVAL_FLAGS(ssa_op->op2_use) |= ZREG_LAST_USE;
2979 }
2980 }
2981 }
2982 if (ssa_op->result_use >= 0
2983 && ssa_op->result_use != ssa_op->op1_use
2984 && ssa_op->result_use != ssa_op->op2_use
2985 && RA_HAS_IVAL(ssa_op->result_use)) {
2986 if (!support_opline) {
2987 RA_IVAL_DEL(ssa_op->result_use);
2988 count--;
2989 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2990 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain,
2991 ra,
2992 ssa, ssa_opcodes, op_array, op_array_ssa);
2993 }
2994 }
2995
2996 if (ssa_op->op1_def >= 0) {
2997 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
2998 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2999 }
3000 if (ssa_op->op2_def >= 0) {
3001 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op2.var), idx);
3002 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
3003 }
3004 if (ssa_op->result_def >= 0) {
3005 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->result.var), idx);
3006 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3007 }
3008
3009 if (support_opline) {
3010 if (ssa_op->result_def >= 0
3011 && (ssa->vars[ssa_op->result_def].use_chain >= 0
3012 || ssa->vars[ssa_op->result_def].phi_use_chain)
3013 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
3014 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
3015 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
3016 || opline->opcode == ZEND_PRE_INC
3017 || opline->opcode == ZEND_PRE_DEC
3018 || opline->opcode == ZEND_POST_INC
3019 || opline->opcode == ZEND_POST_DEC
3020 || opline->opcode == ZEND_ADD
3021 || opline->opcode == ZEND_SUB
3022 || opline->opcode == ZEND_MUL
3023 || opline->opcode == ZEND_FETCH_DIM_R
3024 || opline->opcode == ZEND_FETCH_OBJ_R
3025 || opline->opcode == ZEND_FETCH_CONSTANT) {
3026 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
3027 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
3028 vars_op_array[ssa_op->result_def] = op_array;
3029 RA_IVAL_START(ssa_op->result_def, idx);
3030 count++;
3031 }
3032 }
3033 }
3034 if (ssa_op->op1_def >= 0
3035 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
3036 || ssa->vars[ssa_op->op1_def].phi_use_chain)
3037 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
3038 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)
3039 && (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
3040 || opline->opcode == ZEND_PRE_INC
3041 || opline->opcode == ZEND_PRE_DEC
3042 || opline->opcode == ZEND_POST_INC
3043 || opline->opcode == ZEND_POST_DEC)) {
3044 vars_op_array[ssa_op->op1_def] = op_array;
3045 RA_IVAL_START(ssa_op->op1_def, idx);
3046 count++;
3047 }
3048 if (ssa_op->op2_def >= 0
3049 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
3050 || ssa->vars[ssa_op->op2_def].phi_use_chain)
3051 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
3052 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)
3053 && !(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)) {
3054 vars_op_array[ssa_op->op2_def] = op_array;
3055 RA_IVAL_START(ssa_op->op2_def, idx);
3056 count++;
3057 }
3058 }
3059
3060 len = zend_jit_trace_op_len(opline);
3061 switch (opline->opcode) {
3062 case ZEND_ASSIGN_DIM:
3063 case ZEND_ASSIGN_OBJ:
3064 case ZEND_ASSIGN_STATIC_PROP:
3065 case ZEND_ASSIGN_DIM_OP:
3066 case ZEND_ASSIGN_OBJ_OP:
3067 case ZEND_ASSIGN_STATIC_PROP_OP:
3068 case ZEND_ASSIGN_OBJ_REF:
3069 case ZEND_ASSIGN_STATIC_PROP_REF:
3070 case ZEND_FRAMELESS_ICALL_3:
3071 /* OP_DATA */
3072 ssa_op++;
3073 opline++;
3074 if (ssa_op->op1_use >= 0
3075 && RA_HAS_IVAL(ssa_op->op1_use)
3076 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
3077 if (support_opline) {
3078 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain,
3079 ra,
3080 ssa, ssa_opcodes, op_array, op_array_ssa);
3081 if (opline->op1_type != IS_CV) {
3082 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_LAST_USE;
3083 }
3084 } else {
3085 RA_IVAL_DEL(ssa_op->op1_use);
3086 count--;
3087 }
3088 }
3089 if (ssa_op->op1_def >= 0) {
3090 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3091 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3092 if (support_opline
3093 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
3094 || ssa->vars[ssa_op->op1_def].phi_use_chain)
3095 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
3096 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)
3097 && !(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)) {
3098 vars_op_array[ssa_op->op1_def] = op_array;
3099 RA_IVAL_START(ssa_op->op1_def, idx);
3100 count++;
3101 }
3102 }
3103 ssa_op++;
3104 opline++;
3105 idx+=2;
3106 break;
3107 case ZEND_RECV_INIT:
3108 ssa_op++;
3109 opline++;
3110 idx++;
3111 while (opline->opcode == ZEND_RECV_INIT) {
3112 /* RECV_INIT doesn't support registers */
3113 if (ssa_op->result_use >= 0 && RA_HAS_IVAL(ssa_op->result_use)) {
3114 RA_IVAL_DEL(ssa_op->result_use);
3115 count--;
3116 }
3117 if (ssa_op->result_def >= 0) {
3118 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->result.var), idx);
3119 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3120 }
3121 ssa_op++;
3122 opline++;
3123 idx++;
3124 }
3125 break;
3126 case ZEND_BIND_GLOBAL:
3127 ssa_op++;
3128 opline++;
3129 idx++;
3130 while (opline->opcode == ZEND_BIND_GLOBAL) {
3131 /* BIND_GLOBAL doesn't support registers */
3132 if (ssa_op->op1_def >= 0) {
3133 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3134 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3135 }
3136 ssa_op++;
3137 opline++;
3138 idx++;
3139 }
3140 break;
3141 default:
3142 ssa_op += len;
3143 idx += len;
3144 break;
3145 }
3146 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3147 /* New call frames */
3148 zend_jit_trace_stack_frame *prev_frame = frame;
3149
3150 /* Clear allocated registers */
3151 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3152 j = STACK_VAR(stack, i);
3153 if (j >= 0 && RA_HAS_IVAL(j) && !(RA_IVAL_FLAGS(j) & ZREG_LAST_USE)) {
3154 RA_IVAL_DEL(j);
3155 count--;
3156 }
3157 }
3158
3159 frame = zend_jit_trace_call_frame(frame, op_array);
3160 frame->prev = prev_frame;
3161 frame->func = (const zend_function*)p->op_array;
3162 stack = frame->stack;
3163 op_array = p->op_array;
3164 jit_extension =
3165 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3166 op_array_ssa = &jit_extension->func_info.ssa;
3167 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3168 for (i = 0; i < op_array->last_var; i++) {
3169 SET_STACK_VAR(stack, i, j);
3170 vars_op_array[j] = op_array;
3171 if (ssa->vars[j].use_chain >= 0
3172 && ssa->vars[j].alias == NO_ALIAS
3173 && zend_jit_var_supports_reg(ssa, j)) {
3174 RA_IVAL_START(j, idx);
3175 RA_IVAL_FLAGS(j) = ZREG_LOAD;
3176 count++;
3177 }
3178 j++;
3179 }
3180 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3181 SET_STACK_VAR(stack, i, -1);
3182 }
3183 level++;
3184 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3185 /* Close exiting call frames */
3186 for (i = 0; i < op_array->last_var; i++) {
3187 RA_IVAL_CLOSE(i, idx-1);
3188 }
3189 op_array = p->op_array;
3190 jit_extension =
3191 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3192 op_array_ssa = &jit_extension->func_info.ssa;
3193 frame = zend_jit_trace_ret_frame(frame, op_array);
3194 stack = frame->stack;
3195 if (level == 0) {
3196 /* New return frames */
3197 frame->prev = NULL;
3198 frame->func = (const zend_function*)op_array;
3199 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3200 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3201 SET_STACK_VAR(stack, i, j);
3202 vars_op_array[j] = op_array;
3203 if (ssa->vars[j].use_chain >= 0
3204 && ssa->vars[j].alias == NO_ALIAS
3205 && zend_jit_var_supports_reg(ssa, j)
3206 && !(ssa->var_info[j].type & MAY_BE_GUARD)) {
3207 RA_IVAL_START(j, idx);
3208 RA_IVAL_FLAGS(j) = ZREG_LOAD;
3209 count++;
3210 }
3211 j++;
3212 }
3213 } else {
3214 level--;
3215 }
3216 } else if (p->op == ZEND_JIT_TRACE_END) {
3217 break;
3218 }
3219 }
3220
3221 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3222 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3223 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3224 zend_ssa_phi *phi = ssa->blocks[1].phis;
3225
3226 while (phi) {
3227 i = phi->sources[1];
3228 if (RA_HAS_IVAL(i) && !ssa->vars[phi->ssa_var].no_val) {
3229 RA_IVAL_END(i, idx);
3230 RA_IVAL_FLAGS(i) &= ~ZREG_LAST_USE;
3231 }
3232 phi = phi->next;
3233 }
3234
3235 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3236 for (i = 0; i < op_array->last_var; i++) {
3237 if (RA_HAS_IVAL(i) && !ssa->vars[i].phi_use_chain) {
3238 RA_IVAL_END(i, idx);
3239 RA_IVAL_FLAGS(i) &= ~ZREG_LAST_USE;
3240 } else {
3241 RA_IVAL_CLOSE(i, idx);
3242 }
3243 }
3244 }
3245 }
3246
3247 if (count) {
3248 for (i = 0; i < ssa->vars_count; i++) {
3249 if (RA_HAS_REG(i)) {
3250 if ((RA_REG_FLAGS(i) & ZREG_LOAD) &&
3251 (RA_REG_FLAGS(i) & ZREG_LAST_USE) &&
3252 (i >= parent_vars_count || STACK_REG(parent_stack, i) == ZREG_NONE) &&
3253 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0) {
3254 /* skip life range with single use */
3255 RA_REG_DEL(i);
3256 count--;
3257 }
3258 }
3259 }
3260 }
3261
3262 if (count) {
3263 /* SSA resolution */
3264 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3265 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3266 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3267 zend_ssa_phi *phi = ssa->blocks[1].phis;
3268
3269 while (phi) {
3270 int def = phi->ssa_var;
3271 int use = phi->sources[1];
3272
3273 if (RA_HAS_REG(def)) {
3274 if (!RA_HAS_REG(use)) {
3275 RA_REG_FLAGS(def) |= ZREG_LOAD;
3276 if ((RA_REG_FLAGS(def) & ZREG_LAST_USE)
3277 && ssa->vars[def].use_chain >= 0
3278 && !ssa->vars[def].phi_use_chain
3279 && zend_ssa_next_use(ssa->ops, def, ssa->vars[def].use_chain) < 0
3280 ) {
3281 /* remove interval used once */
3282 RA_REG_DEL(def);
3283 count--;
3284 }
3285 } else if ((ssa->var_info[def].type & MAY_BE_ANY) != (ssa->var_info[use].type & MAY_BE_ANY)) {
3286 RA_REG_FLAGS(def) |= ZREG_LOAD;
3287 RA_REG_FLAGS(use) |= ZREG_STORE;
3288 } else {
3289 use = phi->sources[0];
3290 if (zend_jit_var_supports_reg(ssa, use)) {
3291 ZEND_ASSERT(!RA_HAS_REG(use));
3292 RA_REG_START(use, 0);
3293 RA_REG_FLAGS(use) = ZREG_LOAD;
3294 count++;
3295 } else {
3296 RA_REG_FLAGS(def) |= ZREG_LOAD;
3297 }
3298 }
3299 } else if (RA_HAS_REG(use)) {
3300 if (ssa->vars[use].use_chain >= 0) {
3301 RA_REG_FLAGS(use) |= ZREG_STORE; // TODO: ext/opcache/tests/jit/reg_alloc_00[67].phpt ???
3302 } else {
3303 RA_REG_DEL(use);
3304 count--;
3305 }
3306 }
3307 phi = phi->next;
3308 }
3309 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
3310 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
3311 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3312 int var = STACK_VAR(stack, i);
3313 if (var >= 0 && RA_HAS_REG(var)
3314 && !(RA_REG_FLAGS(var) & (ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE))) {
3315 RA_REG_FLAGS(var) |= ZREG_STORE;
3316 }
3317 }
3318 }
3319
3320 if (!count) {
3321 zend_arena_release(&CG(arena), checkpoint);
3322 return NULL;
3323 }
3324
3325 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3326 fprintf(stderr, "---- TRACE %d Live Ranges \"%s\"\n", ZEND_JIT_TRACE_NUM, op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
3327 for (i = 0; i < ssa->vars_count; i++) {
3328 if (RA_HAS_REG(i)) {
3329 fprintf(stderr, "#%d.", i);
3330 uint32_t var_num = ssa->vars[i].var;
3331 zend_dump_var(vars_op_array[i], (var_num < vars_op_array[i]->last_var ? IS_CV : 0), var_num);
3332 if (RA_REG_FLAGS(i) & ZREG_LAST_USE) {
3333 fprintf(stderr, " last_use");
3334 }
3335 if (RA_REG_FLAGS(i) & ZREG_LOAD) {
3336 fprintf(stderr, " load");
3337 }
3338 if (RA_REG_FLAGS(i) & ZREG_STORE) {
3339 fprintf(stderr, " store");
3340 }
3341 fprintf(stderr, "\n");
3342 }
3343 }
3344 fprintf(stderr, "\n");
3345 }
3346
3347 return ra;
3348 }
3349
3350 zend_arena_release(&CG(arena), checkpoint);
3351 return NULL;
3352 }
3353
zend_jit_trace_cleanup_stack(zend_jit_ctx * jit,zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes)3354 static void zend_jit_trace_cleanup_stack(zend_jit_ctx *jit, zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes)
3355 {
3356 if (ssa_op->op1_use >= 0
3357 && jit->ra[ssa_op->op1_use].ref
3358 && (jit->ra[ssa_op->op1_use].flags & ZREG_LAST_USE)
3359 && (ssa_op->op1_use_chain == -1
3360 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->op1_use_chain], ssa->ops + ssa_op->op1_use_chain, ssa_op->op1_use))) {
3361 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var));
3362 }
3363 if (ssa_op->op2_use >= 0
3364 && ssa_op->op2_use != ssa_op->op1_use
3365 && jit->ra[ssa_op->op2_use].ref
3366 && (jit->ra[ssa_op->op2_use].flags & ZREG_LAST_USE)
3367 && (ssa_op->op2_use_chain == -1
3368 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->op2_use_chain], ssa->ops + ssa_op->op2_use_chain, ssa_op->op2_use))) {
3369 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->op2.var));
3370 }
3371 if (ssa_op->result_use >= 0
3372 && ssa_op->result_use != ssa_op->op1_use
3373 && ssa_op->result_use != ssa_op->op2_use
3374 && jit->ra[ssa_op->result_use].ref
3375 && (jit->ra[ssa_op->result_use].flags & ZREG_LAST_USE)
3376 && (ssa_op->res_use_chain == -1
3377 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->res_use_chain], ssa->ops + ssa_op->res_use_chain, ssa_op->result_use))) {
3378 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
3379 }
3380 }
3381
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3382 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3383 {
3384 zend_op *next_opline = (zend_op*)(opline + 1);
3385
3386 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3387 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3388 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3389 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3390 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3391 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3392 }
3393 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3394 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3395 }
3396 }
3397
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3398 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3399 {
3400 int var = ssa_op->result_def;
3401 int i;
3402 int use = ssa->vars[var].use_chain;
3403 const zend_op *opline;
3404
3405 if (use < 0
3406 || ssa->vars[var].phi_use_chain
3407 || ssa->ops[use].op1_use != var
3408 || ssa->ops[use].op1_use_chain != -1) {
3409 return 0;
3410 }
3411
3412 opline = ssa_opcodes[use];
3413 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3414 return (opline->op2_type == IS_CONST &&
3415 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3416 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3417 if (!JIT_G(current_frame)
3418 || !JIT_G(current_frame)->call
3419 || !JIT_G(current_frame)->call->func
3420 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3421 return 0;
3422 }
3423 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3424 && opline->opcode != ZEND_FETCH_OBJ_IS
3425 && opline->opcode != ZEND_FETCH_OBJ_W
3426 && opline->opcode != ZEND_ASSIGN_OBJ
3427 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3428 && opline->opcode != ZEND_PRE_INC_OBJ
3429 && opline->opcode != ZEND_PRE_DEC_OBJ
3430 && opline->opcode != ZEND_POST_INC_OBJ
3431 && opline->opcode != ZEND_POST_DEC_OBJ) {
3432 return 0;
3433 }
3434
3435 if (opline->op2_type != IS_CONST
3436 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3437 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3438 return 0;
3439 }
3440
3441 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3442 if (opline->op1_type == IS_CV
3443 && (opline+1)->op1_type == IS_CV
3444 && (opline+1)->op1.var == opline->op1.var) {
3445 /* skip $a->prop += $a; */
3446 return 0;
3447 }
3448 if (!zend_jit_supported_binary_op(
3449 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3450 return 0;
3451 }
3452 }
3453
3454 for (i = ssa->vars[var].definition; i < use; i++) {
3455 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3456 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3457 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3458 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3459 return 0;
3460 }
3461 }
3462
3463 return 1;
3464 }
3465
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3466 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3467 {
3468 uint32_t i;
3469
3470 for (i = 0; i < stack_size; i++) {
3471 if (STACK_FLAGS(stack, i) & ~(ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE)) {
3472 return 1;
3473 } else if (STACK_REG(stack, i) != ZREG_NONE) {
3474 return 1;
3475 }
3476 }
3477 return 0;
3478 }
3479
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3480 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3481 {
3482 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3483 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3484 uint32_t stack_size;
3485 zend_jit_trace_stack *stack;
3486
3487 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3488 return 1;
3489 }
3490
3491 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3492 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3493 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3494 }
3495
zend_jit_trace_deoptimization(zend_jit_ctx * jit,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_jit_exit_const * constants,int8_t func_reg,bool polymorphic_side_trace)3496 static int zend_jit_trace_deoptimization(
3497 zend_jit_ctx *jit,
3498 uint32_t flags,
3499 const zend_op *opline,
3500 zend_jit_trace_stack *parent_stack,
3501 int parent_vars_count,
3502 zend_ssa *ssa,
3503 zend_jit_trace_stack *stack,
3504 zend_jit_exit_const *constants,
3505 int8_t func_reg,
3506 bool polymorphic_side_trace)
3507 {
3508 int i;
3509 int check2 = -1;
3510
3511 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3512 for (i = 0; i < parent_vars_count; i++) {
3513 int8_t reg = STACK_REG(parent_stack, i);
3514
3515 if (STACK_FLAGS(parent_stack, i) == ZREG_CONST) {
3516 uint8_t type = STACK_TYPE(parent_stack, i);
3517
3518 if (type == IS_LONG) {
3519 if (!zend_jit_store_const_long(jit, i,
3520 (zend_long)constants[STACK_REF(parent_stack, i)].i)) {
3521 return 0;
3522 }
3523 } else if (type == IS_DOUBLE) {
3524 if (!zend_jit_store_const_double(jit, i,
3525 constants[STACK_REF(parent_stack, i)].d)) {
3526 return 0;
3527 }
3528 } else {
3529 ZEND_UNREACHABLE();
3530 }
3531 if (stack) {
3532 SET_STACK_TYPE(stack, i, type, 1);
3533 if (jit->ra && jit->ra[i].ref) {
3534 SET_STACK_REF(stack, i, jit->ra[i].ref);
3535 }
3536 }
3537 } else if (STACK_FLAGS(parent_stack, i) == ZREG_TYPE_ONLY) {
3538 uint8_t type = STACK_TYPE(parent_stack, i);
3539
3540 if (!zend_jit_store_type(jit, i, type)) {
3541 return 0;
3542 }
3543 if (stack) {
3544 SET_STACK_TYPE(stack, i, type, 1);
3545 }
3546 } else if (STACK_FLAGS(parent_stack, i) == ZREG_THIS) {
3547 if (polymorphic_side_trace) {
3548 ssa->var_info[i].delayed_fetch_this = 1;
3549 if (stack) {
3550 SET_STACK_REG_EX(stack, i, ZREG_NONE, ZREG_THIS);
3551 }
3552 } else if (!zend_jit_load_this(jit, EX_NUM_TO_VAR(i))) {
3553 return 0;
3554 }
3555 } else if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_ADDREF) {
3556 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(i));
3557 zend_jit_zval_try_addref(jit, dst);
3558 } else if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
3559 ZEND_ASSERT(reg != ZREG_NONE);
3560 ZEND_ASSERT(check2 == -1);
3561 check2 = i;
3562 } else if (STACK_FLAGS(parent_stack, i) & ZREG_SPILL_SLOT) {
3563 if (ssa && ssa->vars[i].no_val) {
3564 /* pass */
3565 } else {
3566 uint8_t type = STACK_TYPE(parent_stack, i);
3567
3568 if (!zend_jit_store_spill_slot(jit, 1 << type, i, reg, STACK_REF(parent_stack, i),
3569 STACK_MEM_TYPE(parent_stack, i) != type)) {
3570 return 0;
3571 }
3572 if (stack) {
3573 if (jit->ra && jit->ra[i].ref) {
3574 SET_STACK_TYPE(stack, i, type, 0);
3575 if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
3576 SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
3577 } else {
3578 SET_STACK_REF(stack, i, jit->ra[i].ref);
3579 }
3580 } else {
3581 SET_STACK_TYPE(stack, i, type, 1);
3582 }
3583 }
3584 }
3585 } else if (reg != ZREG_NONE) {
3586 if (ssa && ssa->vars[i].no_val) {
3587 /* pass */
3588 } else {
3589 uint8_t type = STACK_TYPE(parent_stack, i);
3590
3591 if (!zend_jit_store_reg(jit, 1 << type, i, reg,
3592 (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0,
3593 STACK_MEM_TYPE(parent_stack, i) != type)) {
3594 return 0;
3595 }
3596 if (stack) {
3597 if (jit->ra && jit->ra[i].ref) {
3598 SET_STACK_TYPE(stack, i, type, 0);
3599 if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
3600 SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
3601 } else {
3602 SET_STACK_REF(stack, i, jit->ra[i].ref);
3603 }
3604 } else {
3605 SET_STACK_TYPE(stack, i, type, 1);
3606 }
3607 }
3608 }
3609 }
3610 }
3611
3612 if (check2 != -1) {
3613 int8_t reg = STACK_REG(parent_stack, check2);
3614
3615 ZEND_ASSERT(STACK_FLAGS(parent_stack, check2) == ZREG_ZVAL_COPY);
3616 ZEND_ASSERT(reg != ZREG_NONE);
3617 if (!zend_jit_escape_if_undef(jit, check2, flags, opline, reg)) {
3618 return 0;
3619 }
3620 if (!zend_jit_restore_zval(jit, EX_NUM_TO_VAR(check2), reg)) {
3621 return 0;
3622 }
3623 }
3624
3625 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3626 if (!zend_jit_save_call_chain(jit, -1)) {
3627 return 0;
3628 }
3629 }
3630
3631 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3632 const zend_op *op = opline - 1;
3633
3634 if (!zend_jit_free_op(jit, op, -1, op->op2.var)) {
3635 return 0;
3636 }
3637 }
3638
3639 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3640 const zend_op *op = opline - 1;
3641
3642 if (!zend_jit_free_op(jit, op, -1, op->op1.var)) {
3643 return 0;
3644 }
3645 }
3646
3647 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3648 zend_jit_check_exception(jit);
3649 }
3650
3651 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3652 if (!zend_jit_free_trampoline(jit, func_reg)) {
3653 return 0;
3654 }
3655 }
3656
3657 return 1;
3658 }
3659
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3660 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3661 {
3662 info->has_range = 1;
3663 info->range.min = min;
3664 info->range.max = max;
3665 info->range.underflow = 0;
3666 info->range.overflow = 0;
3667 }
3668
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3669 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3670 {
3671 zend_long op1_min, op1_max, op2_min, op2_max;
3672
3673 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3674 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3675 return;
3676 }
3677
3678 op1_min = OP1_MIN_RANGE();
3679 op1_max = OP1_MAX_RANGE();
3680 op2_min = OP2_MIN_RANGE();
3681 op2_max = OP2_MAX_RANGE();
3682
3683 switch (opline->opcode) {
3684 case ZEND_IS_EQUAL:
3685 case ZEND_CASE:
3686 case ZEND_IS_IDENTICAL:
3687 case ZEND_CASE_STRICT:
3688 case ZEND_IS_NOT_IDENTICAL:
3689 if (!exit_if_true) {
3690 /* op1 == op2 */
3691 if (ssa_op->op1_use >= 0) {
3692 zend_jit_trace_set_var_range(
3693 &ssa->var_info[ssa_op->op1_use],
3694 MAX(op1_min, op2_min),
3695 MIN(op1_max, op2_max));
3696 }
3697 if (ssa_op->op2_use >= 0) {
3698 zend_jit_trace_set_var_range(
3699 &ssa->var_info[ssa_op->op2_use],
3700 MAX(op2_min, op1_min),
3701 MIN(op2_max, op1_max));
3702 }
3703 }
3704 break;
3705 case ZEND_IS_NOT_EQUAL:
3706 if (exit_if_true) {
3707 /* op1 == op2 */
3708 if (ssa_op->op1_use >= 0) {
3709 zend_jit_trace_set_var_range(
3710 &ssa->var_info[ssa_op->op1_use],
3711 MAX(op1_min, op2_min),
3712 MIN(op1_max, op2_max));
3713 }
3714 if (ssa_op->op2_use >= 0) {
3715 zend_jit_trace_set_var_range(
3716 &ssa->var_info[ssa_op->op2_use],
3717 MAX(op2_min, op1_min),
3718 MIN(op2_max, op1_max));
3719 }
3720 }
3721 break;
3722 case ZEND_IS_SMALLER_OR_EQUAL:
3723 if (!exit_if_true) {
3724 /* op1 <= op2 */
3725 if (ssa_op->op1_use >= 0) {
3726 zend_jit_trace_set_var_range(
3727 &ssa->var_info[ssa_op->op1_use],
3728 op1_min,
3729 MIN(op1_max, op2_max));
3730 }
3731 if (ssa_op->op2_use >= 0) {
3732 zend_jit_trace_set_var_range(
3733 &ssa->var_info[ssa_op->op2_use],
3734 MAX(op2_min, op1_min),
3735 op2_max);
3736 }
3737 } else {
3738 /* op1 > op2 */
3739 if (ssa_op->op1_use >= 0) {
3740 zend_jit_trace_set_var_range(
3741 &ssa->var_info[ssa_op->op1_use],
3742 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3743 op1_max);
3744 }
3745 if (ssa_op->op2_use >= 0) {
3746 zend_jit_trace_set_var_range(
3747 &ssa->var_info[ssa_op->op2_use],
3748 op2_min,
3749 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3750 }
3751 }
3752 break;
3753 case ZEND_IS_SMALLER:
3754 if (!exit_if_true) {
3755 /* op1 < op2 */
3756 if (ssa_op->op1_use >= 0) {
3757 zend_jit_trace_set_var_range(
3758 &ssa->var_info[ssa_op->op1_use],
3759 op1_min,
3760 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3761 }
3762 if (ssa_op->op2_use >= 0) {
3763 zend_jit_trace_set_var_range(
3764 &ssa->var_info[ssa_op->op2_use],
3765 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3766 op2_max);
3767 }
3768 } else {
3769 /* op1 >= op2 */
3770 if (ssa_op->op1_use >= 0) {
3771 zend_jit_trace_set_var_range(
3772 &ssa->var_info[ssa_op->op1_use],
3773 MAX(op1_min, op2_min),
3774 op1_max);
3775 }
3776 if (ssa_op->op2_use >= 0) {
3777 zend_jit_trace_set_var_range(
3778 &ssa->var_info[ssa_op->op2_use],
3779 op2_min,
3780 MIN(op2_max, op1_max));
3781 }
3782 }
3783 break;
3784 }
3785 }
3786
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3787 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3788 {
3789 uint8_t prev_opcode;
3790
3791 if (opline->op1_type == IS_CONST
3792 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3793 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3794 if (ssa_op->op2_use >= 0) {
3795 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3796 ssa_op--;
3797 opline = ssa_opcodes[ssa_op - ssa->ops];
3798 prev_opcode = opline->opcode;
3799 if (prev_opcode == ZEND_PRE_INC
3800 || prev_opcode == ZEND_PRE_DEC
3801 || prev_opcode == ZEND_POST_INC
3802 || prev_opcode == ZEND_POST_DEC) {
3803 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3804 }
3805 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3806 ssa_op--;
3807 opline = ssa_opcodes[ssa_op - ssa->ops];
3808 prev_opcode = opline->opcode;
3809 if (prev_opcode == ZEND_ADD
3810 || prev_opcode == ZEND_SUB) {
3811 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3812 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3813 }
3814 }
3815 }
3816 } else if (opline->op2_type == IS_CONST
3817 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3818 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3819 if (ssa_op->op1_use >= 0) {
3820 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3821 ssa_op--;
3822 opline = ssa_opcodes[ssa_op - ssa->ops];
3823 prev_opcode = opline->opcode;
3824 if (prev_opcode == ZEND_PRE_INC
3825 || prev_opcode == ZEND_PRE_DEC
3826 || prev_opcode == ZEND_POST_INC
3827 || prev_opcode == ZEND_POST_DEC) {
3828 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3829 }
3830 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3831 ssa_op--;
3832 opline = ssa_opcodes[ssa_op - ssa->ops];
3833 prev_opcode = opline->opcode;
3834 if (prev_opcode == ZEND_ADD
3835 || prev_opcode == ZEND_SUB) {
3836 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3837 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3838 }
3839 }
3840 }
3841 } else {
3842 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3843 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3844
3845 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3846 && prev_ssa_op != ssa->ops
3847 && prev_ssa_op->op1_use >= 0
3848 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3849 prev_ssa_op--;
3850 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3851 }
3852
3853 if (ssa_op->op1_use == prev_ssa_op->op1_use
3854 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3855 if (prev_opcode == ZEND_IS_EQUAL
3856 || prev_opcode == ZEND_IS_NOT_EQUAL
3857 || prev_opcode == ZEND_IS_SMALLER
3858 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3859 || prev_opcode == ZEND_CASE
3860 || prev_opcode == ZEND_IS_IDENTICAL
3861 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3862 || prev_opcode == ZEND_CASE_STRICT) {
3863 if (ssa_op->op1_use < 0) {
3864 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3865 return 0;
3866 }
3867 }
3868 if (ssa_op->op2_use < 0) {
3869 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3870 return 0;
3871 }
3872 }
3873 return 1;
3874 }
3875 }
3876 }
3877 return 0;
3878 }
3879
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3880 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3881 zend_jit_trace_rec *p,
3882 zend_jit_trace_stack_frame *frame)
3883 {
3884 if (opline->result_type == IS_TMP_VAR
3885 && (p+1)->op == ZEND_JIT_TRACE_VM
3886 && (p+1)->opline == opline + 1
3887 && ((opline+1)->opcode == ZEND_SEND_VAL
3888 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3889 && frame
3890 && frame->call
3891 && frame->call->func
3892 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3893 && (opline+1)->op1_type == IS_TMP_VAR
3894 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3895 && (opline+1)->op1.var == opline->result.var) {
3896
3897 if (frame->call && frame->call->func) {
3898 uint8_t res_type = (p+1)->op1_type;
3899
3900 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3901 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3902 }
3903 }
3904 return 1;
3905 }
3906 return 0;
3907 }
3908
zend_jit_find_ssa_var(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t opline_num,uint32_t var_num)3909 static int zend_jit_find_ssa_var(const zend_op_array *op_array,
3910 const zend_ssa *ssa,
3911 uint32_t opline_num,
3912 uint32_t var_num)
3913 {
3914 int ssa_var, j, b = ssa->cfg.map[opline_num];
3915 const zend_basic_block *bb = ssa->cfg.blocks + b;
3916 const zend_ssa_phi *phi;
3917 const zend_ssa_op *ssa_op;
3918 zend_worklist worklist;
3919 ALLOCA_FLAG(use_heap)
3920
3921 while (1) {
3922 ssa_op = ssa->ops + opline_num;
3923 ssa_var = ssa_op->result_def;
3924 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3925 return ssa_var;
3926 }
3927 ssa_var = ssa_op->op2_def;
3928 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3929 return ssa_var;
3930 }
3931 ssa_var = ssa_op->op1_def;
3932 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3933 return ssa_var;
3934 }
3935 if (opline_num == bb->start) {
3936 break;
3937 }
3938 opline_num--;
3939 }
3940 phi = ssa->blocks[b].phis;
3941 ssa_var = -1;
3942 while (phi) {
3943 if (phi->var == var_num) {
3944 ssa_var = phi->ssa_var;
3945 }
3946 phi = phi->next;
3947 }
3948 if (ssa_var >= 0) {
3949 return ssa_var;
3950 }
3951
3952 if (!bb->predecessors_count) {
3953 return -1;
3954 }
3955
3956 ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
3957
3958 for (j = 0; j < bb->predecessors_count; j++) {
3959 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
3960 zend_worklist_push(&worklist, b);
3961 }
3962
3963 while (zend_worklist_len(&worklist) != 0) {
3964 b = zend_worklist_pop(&worklist);
3965 bb = &ssa->cfg.blocks[b];
3966 if (bb->len) {
3967 opline_num = bb->start + bb->len - 1;
3968 while (1) {
3969 ssa_op = ssa->ops + opline_num;
3970 ssa_var = ssa_op->result_def;
3971 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3972 goto found;
3973 }
3974 ssa_var = ssa_op->op2_def;
3975 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3976 goto found;
3977 }
3978 ssa_var = ssa_op->op1_def;
3979 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3980 goto found;
3981 }
3982 if (opline_num == bb->start) {
3983 break;
3984 }
3985 opline_num--;
3986 }
3987 }
3988 phi = ssa->blocks[b].phis;
3989 ssa_var = -1;
3990 while (phi) {
3991 if (phi->var == var_num) {
3992 ssa_var = phi->ssa_var;
3993 }
3994 phi = phi->next;
3995 }
3996 if (ssa_var >= 0) {
3997 goto found;
3998 }
3999 for (j = 0; j < bb->predecessors_count; j++) {
4000 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
4001 zend_worklist_push(&worklist, b);
4002 }
4003 }
4004 ssa_var = -1;
4005
4006 found:
4007 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
4008 return ssa_var;
4009 }
4010
zend_jit_trace_must_store_type(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t opline_num,uint32_t var_num,uint8_t type)4011 static bool zend_jit_trace_must_store_type(const zend_op_array *op_array,
4012 const zend_ssa *ssa,
4013 uint32_t opline_num,
4014 uint32_t var_num,
4015 uint8_t type)
4016 {
4017 if (ssa->var_info) {
4018 int ssa_var = zend_jit_find_ssa_var(op_array, ssa, opline_num, var_num);
4019
4020 if (ssa_var >= 0) {
4021 if ((ssa->var_info[ssa_var].type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1U << type)) {
4022 return 0;
4023 }
4024 }
4025 }
4026 return 1;
4027 }
4028
zend_jit_trace_may_throw(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,const zend_ssa * ssa,uint32_t t1,uint32_t t2,uint32_t t3,uint32_t val_type)4029 static bool zend_jit_trace_may_throw(const zend_op *opline,
4030 const zend_ssa_op *ssa_op,
4031 const zend_op_array *op_array,
4032 const zend_ssa *ssa,
4033 uint32_t t1,
4034 uint32_t t2,
4035 uint32_t t3,
4036 uint32_t val_type)
4037 {
4038 switch (opline->opcode) {
4039 case ZEND_ASSIGN_DIM_OP:
4040 if (opline->extended_value != ZEND_CONCAT
4041 && val_type == IS_LONG
4042 && (t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_ARRAY
4043 && MAY_BE_PACKED_ONLY(t1)
4044 && !(t1 & MAY_BE_ARRAY_OF_REF)
4045 && (t2 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_LONG
4046 && (t3 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_LONG) {
4047 return 0;
4048 }
4049 break;
4050 default:
4051 break;
4052 }
4053 return zend_may_throw_ex(opline, ssa_op, op_array, ssa, t1, t2);
4054 }
4055
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)4056 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
4057 {
4058 const void *handler = NULL;
4059 zend_jit_ctx ctx;
4060 zend_jit_ctx *jit = &ctx;
4061 zend_jit_reg_var *ra = NULL;
4062 zend_script *script = NULL;
4063 zend_string *name = NULL;
4064 void *checkpoint;
4065 const zend_op_array *op_array;
4066 zend_ssa *ssa, *op_array_ssa;
4067 const zend_op **ssa_opcodes;
4068 zend_jit_trace_rec *p;
4069 zend_jit_op_array_trace_extension *jit_extension;
4070 int num_op_arrays = 0;
4071 zend_jit_trace_info *t;
4072 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
4073 uint8_t smart_branch_opcode;
4074 const void *exit_addr;
4075 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info, op1_mem_info;
4076 bool send_result = 0;
4077 bool skip_comparison;
4078 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
4079 zend_class_entry *ce;
4080 bool ce_is_instanceof;
4081 bool on_this = 0;
4082 bool delayed_fetch_this = 0;
4083 bool avoid_refcounting = 0;
4084 bool polymorphic_side_trace =
4085 parent_trace &&
4086 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
4087 uint32_t i;
4088 zend_jit_trace_stack_frame *frame, *top, *call;
4089 zend_jit_trace_stack *stack;
4090 uint8_t res_type = IS_UNKNOWN;
4091 const zend_op *opline, *orig_opline;
4092 const zend_ssa_op *ssa_op, *orig_ssa_op;
4093 int checked_stack;
4094 int peek_checked_stack;
4095 uint32_t frame_flags = 0;
4096
4097 JIT_G(current_trace) = trace_buffer;
4098
4099 checkpoint = zend_arena_checkpoint(CG(arena));
4100
4101 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4102
4103 if (!ssa) {
4104 goto jit_cleanup;
4105 }
4106
4107 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4108
4109 op_array = trace_buffer->op_array;
4110 opline = trace_buffer[1].opline;
4111 name = zend_jit_trace_name(op_array, opline->lineno);
4112 zend_jit_trace_start(&ctx, op_array, ssa, name, ZEND_JIT_TRACE_NUM,
4113 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4114 ctx.trace = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
4115
4116 /* Register allocation */
4117 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4118 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4119 ctx.ra = ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4120 }
4121
4122 p = trace_buffer;
4123 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4124 op_array = p->op_array;
4125 frame = JIT_G(current_frame);
4126 top = zend_jit_trace_call_frame(frame, op_array);
4127 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4128 frame->used_stack = checked_stack = peek_checked_stack = 0;
4129 stack = frame->stack;
4130 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4131 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4132 }
4133
4134 opline = p[1].opline;
4135 p += ZEND_JIT_TRACE_START_REC_SIZE;
4136
4137 jit_extension =
4138 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4139 op_array_ssa = &jit_extension->func_info.ssa;
4140
4141 if (!parent_trace) {
4142 zend_jit_set_last_valid_opline(&ctx, opline);
4143 zend_jit_track_last_valid_opline(&ctx);
4144 } else {
4145 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4146 zend_jit_trace_opline_guard(&ctx, opline);
4147 } else {
4148 zend_jit_reset_last_valid_opline(&ctx);
4149 }
4150 }
4151
4152 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4153 int last_var;
4154 int parent_vars_count = 0;
4155 zend_jit_trace_stack *parent_stack = NULL;
4156 int used_stack = ((zend_tssa*)ssa)->used_stack;
4157
4158 if (used_stack > 0) {
4159 peek_checked_stack = used_stack;
4160 if (!zend_jit_stack_check(&ctx, opline, used_stack)) {
4161 goto jit_failure;
4162 }
4163 }
4164
4165 if (parent_trace) {
4166 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4167 op_array->last_var + op_array->T);
4168 if (parent_vars_count) {
4169 parent_stack =
4170 zend_jit_traces[parent_trace].stack_map +
4171 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4172 }
4173 }
4174
4175 last_var = op_array->last_var;
4176 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4177 last_var += op_array->T;
4178 }
4179
4180 for (i = 0; i < last_var; i++) {
4181 uint32_t info = ssa->var_info[i].type;
4182
4183 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4184 uint8_t type, mem_type;
4185
4186 type = concrete_type(info);
4187 if (i < parent_vars_count
4188 && STACK_TYPE(parent_stack, i) == type) {
4189 mem_type = STACK_MEM_TYPE(parent_stack, i);
4190 if (mem_type != IS_UNKNOWN) {
4191 SET_STACK_TYPE(stack, i, mem_type, 1);
4192 }
4193 SET_STACK_TYPE(stack, i, type, 0);
4194 } else {
4195 SET_STACK_TYPE(stack, i, type, 1);
4196 }
4197 } else if (ssa->vars[i].alias != NO_ALIAS) {
4198 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4199 } else if (i < parent_vars_count
4200 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4201 /* This must be already handled by trace type inference */
4202 ZEND_ASSERT(ssa->vars[i].use_chain < 0 && !ssa->vars[i].phi_use_chain);
4203 SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i), 1);
4204 } else if ((info & MAY_BE_GUARD) != 0
4205 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4206 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4207 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4208 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4209 && (ssa->vars[i].use_chain != -1
4210 || (ssa->vars[i].phi_use_chain
4211 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4212 /* Check loop-invariant variable type */
4213 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4214 goto jit_failure;
4215 }
4216 info &= ~MAY_BE_GUARD;
4217 ssa->var_info[i].type = info;
4218 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4219 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4220 && op_array->function_name
4221 && i >= op_array->num_args) {
4222 /* This must be already handled by trace type inference */
4223 ZEND_UNREACHABLE();
4224 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4225 }
4226
4227 if ((info & MAY_BE_PACKED_GUARD) != 0
4228 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4229 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4230 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4231 && (ssa->vars[i].use_chain != -1
4232 || (ssa->vars[i].phi_use_chain
4233 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4234 if (!zend_jit_packed_guard(&ctx, opline, EX_NUM_TO_VAR(i), info)) {
4235 goto jit_failure;
4236 }
4237 info &= ~MAY_BE_PACKED_GUARD;
4238 ssa->var_info[i].type = info;
4239 }
4240 }
4241
4242 if (parent_trace) {
4243 /* Deoptimization */
4244 if (!zend_jit_trace_deoptimization(&ctx,
4245 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4246 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4247 parent_stack, parent_vars_count, ssa, stack,
4248 zend_jit_traces[parent_trace].constants,
4249 zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg,
4250 polymorphic_side_trace)) {
4251 goto jit_failure;
4252 }
4253 }
4254
4255 if (ra
4256 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4257 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4258 for (i = 0; i < last_var; i++) {
4259 if (RA_HAS_REG(i)
4260 && (RA_REG_FLAGS(i) & ZREG_LOAD) != 0
4261 && ra[i].ref != STACK_REF(stack, i)
4262 ) {
4263
4264 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4265 uint8_t op_type;
4266
4267 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4268 op_type = concrete_type(ssa->var_info[i].type);
4269 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), op_type)) {
4270 goto jit_failure;
4271 }
4272 SET_STACK_TYPE(stack, i, op_type, 1);
4273 }
4274
4275 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
4276 if (!zend_jit_load_var(&ctx, ssa->var_info[i].type, i, i)) {
4277 goto jit_failure;
4278 }
4279 SET_STACK_REF_EX(stack, i, ra[i].ref, ZREG_LOAD);
4280 } else {
4281 SET_STACK_REF_EX(stack, i, IR_NULL, ZREG_LOAD);
4282 }
4283 }
4284 }
4285 }
4286 }
4287
4288 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4289 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4290 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4291
4292 jit->trace_loop_ref = zend_jit_trace_begin_loop(&ctx); /* start of of trace loop */
4293
4294 if (ra) {
4295 zend_ssa_phi *phi = ssa->blocks[1].phis;
4296
4297 /* First try to insert IR Phi */
4298 while (phi) {
4299 if (RA_HAS_REG(phi->ssa_var)
4300 && !(RA_REG_FLAGS(phi->ssa_var) & ZREG_LOAD)) {
4301 zend_jit_trace_gen_phi(&ctx, phi);
4302 SET_STACK_REF(stack, phi->var, ra[phi->ssa_var].ref);
4303 }
4304 phi = phi->next;
4305 }
4306
4307 phi = ssa->blocks[1].phis;
4308 while (phi) {
4309 if (RA_HAS_REG(phi->ssa_var)) {
4310 if (RA_REG_FLAGS(phi->ssa_var) & ZREG_LOAD) {
4311 uint32_t info = ssa->var_info[phi->ssa_var].type;
4312
4313 if (info & MAY_BE_GUARD) {
4314 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4315 goto jit_failure;
4316 }
4317 info &= ~MAY_BE_GUARD;
4318 ssa->var_info[phi->ssa_var].type = info;
4319 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4320 }
4321 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
4322 goto jit_failure;
4323 }
4324 SET_STACK_REF_EX(stack, phi->var, ra[phi->ssa_var].ref, ZREG_LOAD);
4325 } else if (RA_REG_FLAGS(phi->ssa_var) & ZREG_STORE) {
4326
4327 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var,
4328 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4329 goto jit_failure;
4330 }
4331 SET_STACK_REF_EX(stack, phi->var, ra[phi->ssa_var].ref, ZREG_STORE);
4332 } else {
4333 /* Register has to be written back on side exit */
4334 SET_STACK_REF(stack, phi->var, ra[phi->ssa_var].ref);
4335 }
4336 }
4337 phi = phi->next;
4338 }
4339 }
4340
4341 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4342 // if (ra && dzend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4343 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4344 //
4345 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4346 // if (!timeout_exit_addr) {
4347 // goto jit_failure;
4348 // }
4349 // }
4350 // }
4351
4352 }
4353
4354 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4355 for (;;p++) {
4356 if (p->op == ZEND_JIT_TRACE_VM) {
4357 uint8_t op1_type = p->op1_type;
4358 uint8_t op2_type = p->op2_type;
4359 uint8_t op3_type = p->op3_type;
4360 uint8_t orig_op1_type = op1_type;
4361 uint8_t orig_op2_type = op2_type;
4362 uint8_t val_type = IS_UNKNOWN;
4363 bool op1_indirect;
4364 zend_class_entry *op1_ce = NULL;
4365 zend_class_entry *op2_ce = NULL;
4366 bool gen_handler = false;
4367
4368 opline = p->opline;
4369 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4370 op1_type = IS_UNKNOWN;
4371 }
4372 if (op1_type != IS_UNKNOWN) {
4373 op1_type &= ~IS_TRACE_PACKED;
4374 }
4375 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4376 op2_type = IS_UNKNOWN;
4377 }
4378 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4379 op3_type = IS_UNKNOWN;
4380 }
4381
4382 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4383 op1_ce = (zend_class_entry*)(p+1)->ce;
4384 p++;
4385 }
4386 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4387 op2_ce = (zend_class_entry*)(p+1)->ce;
4388 p++;
4389 }
4390 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4391 val_type = (p+1)->op1_type;
4392 p++;
4393 }
4394
4395 frame_flags = 0;
4396
4397 switch (opline->opcode) {
4398 case ZEND_INIT_FCALL:
4399 case ZEND_INIT_FCALL_BY_NAME:
4400 case ZEND_INIT_NS_FCALL_BY_NAME:
4401 case ZEND_INIT_METHOD_CALL:
4402 case ZEND_INIT_DYNAMIC_CALL:
4403 case ZEND_INIT_STATIC_METHOD_CALL:
4404 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
4405 case ZEND_INIT_USER_CALL:
4406 case ZEND_NEW:
4407 frame->call_level++;
4408 }
4409
4410 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4411 switch (opline->opcode) {
4412 case ZEND_PRE_INC:
4413 case ZEND_PRE_DEC:
4414 case ZEND_POST_INC:
4415 case ZEND_POST_DEC:
4416 if (opline->op1_type != IS_CV) {
4417 break;
4418 }
4419 op1_info = OP1_INFO();
4420 CHECK_OP1_TRACE_TYPE();
4421 if (!(op1_info & MAY_BE_LONG)) {
4422 break;
4423 }
4424 if (opline->result_type != IS_UNUSED) {
4425 res_use_info = zend_jit_trace_type_to_info(
4426 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4427 if (opline->result_type == IS_CV) {
4428 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4429 }
4430 res_info = RES_INFO();
4431 res_addr = RES_REG_ADDR();
4432 } else {
4433 res_use_info = -1;
4434 res_info = -1;
4435 res_addr = 0;
4436 }
4437 op1_def_info = OP1_DEF_INFO();
4438 if (op1_def_info & MAY_BE_GUARD
4439 && !has_concrete_type(op1_def_info)) {
4440 op1_def_info &= ~MAY_BE_GUARD;
4441 }
4442 if (!zend_jit_inc_dec(&ctx, opline,
4443 op1_info, OP1_REG_ADDR(),
4444 op1_def_info, OP1_DEF_REG_ADDR(),
4445 res_use_info, res_info,
4446 res_addr,
4447 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4448 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4449 goto jit_failure;
4450 }
4451 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4452 && !(op1_info & MAY_BE_STRING)) {
4453 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4454 if (opline->result_type != IS_UNUSED) {
4455 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4456 }
4457 } else if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)
4458 && !(op1_info & MAY_BE_STRING)) {
4459 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4460 if (opline->result_type != IS_UNUSED) {
4461 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4462 }
4463 }
4464 if (opline->result_type != IS_UNUSED
4465 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4466 && !(op1_info & MAY_BE_STRING)) {
4467 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4468 } else if (opline->result_type != IS_UNUSED
4469 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)
4470 && !(res_info & MAY_BE_STRING)) {
4471 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4472 }
4473 goto done;
4474 case ZEND_BW_OR:
4475 case ZEND_BW_AND:
4476 case ZEND_BW_XOR:
4477 case ZEND_SL:
4478 case ZEND_SR:
4479 case ZEND_MOD:
4480 op1_info = OP1_INFO();
4481 CHECK_OP1_TRACE_TYPE();
4482 op2_info = OP2_INFO();
4483 CHECK_OP2_TRACE_TYPE();
4484 if (!(op1_info & MAY_BE_LONG)
4485 || !(op2_info & MAY_BE_LONG)) {
4486 break;
4487 }
4488 res_addr = RES_REG_ADDR();
4489 if (Z_MODE(res_addr) != IS_REG
4490 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4491 send_result = 1;
4492 res_use_info = -1;
4493 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4494 if (!zend_jit_reuse_ip(&ctx)) {
4495 goto jit_failure;
4496 }
4497 } else {
4498 res_use_info = zend_jit_trace_type_to_info(
4499 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4500 if (opline->result_type == IS_CV) {
4501 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4502 }
4503 }
4504 res_info = RES_INFO();
4505 if (!zend_jit_long_math(&ctx, opline,
4506 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4507 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4508 res_use_info, res_info, res_addr,
4509 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4510 goto jit_failure;
4511 }
4512 goto done;
4513 case ZEND_ADD:
4514 case ZEND_SUB:
4515 case ZEND_MUL:
4516 // case ZEND_DIV: // TODO: check for division by zero ???
4517 op1_info = OP1_INFO();
4518 op1_addr = OP1_REG_ADDR();
4519 op2_info = OP2_INFO();
4520 op2_addr = OP2_REG_ADDR();
4521 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4522 break;
4523 }
4524 if (opline->opcode == ZEND_ADD &&
4525 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4526 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4527 /* pass */
4528 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4529 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4530 break;
4531 }
4532 if (orig_op1_type != IS_UNKNOWN
4533 && (orig_op1_type & IS_TRACE_REFERENCE)
4534 && opline->op1_type == IS_CV
4535 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4536 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4537 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4538 goto jit_failure;
4539 }
4540 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4541 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4542 }
4543 } else {
4544 CHECK_OP1_TRACE_TYPE();
4545 }
4546 if (orig_op2_type != IS_UNKNOWN
4547 && (orig_op2_type & IS_TRACE_REFERENCE)
4548 && opline->op2_type == IS_CV
4549 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4550 if (!zend_jit_fetch_reference(&ctx, opline, orig_op2_type, &op2_info, &op2_addr,
4551 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4552 goto jit_failure;
4553 }
4554 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4555 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4556 }
4557 } else {
4558 CHECK_OP2_TRACE_TYPE();
4559 }
4560 res_addr = RES_REG_ADDR();
4561 if (Z_MODE(res_addr) != IS_REG
4562 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4563 send_result = 1;
4564 res_use_info = -1;
4565 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4566 if (!zend_jit_reuse_ip(&ctx)) {
4567 goto jit_failure;
4568 }
4569 } else {
4570 res_use_info = zend_jit_trace_type_to_info(
4571 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4572 if (opline->result_type == IS_CV) {
4573 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4574 }
4575 }
4576 res_info = RES_INFO();
4577 if (opline->opcode == ZEND_ADD &&
4578 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4579 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4580 if (!zend_jit_add_arrays(&ctx, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4581 goto jit_failure;
4582 }
4583 } else {
4584 bool may_overflow = (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa);
4585
4586 if (ra
4587 && may_overflow
4588 && ((res_info & MAY_BE_GUARD)
4589 && (res_info & MAY_BE_ANY) == MAY_BE_LONG)
4590 && ((opline->opcode == ZEND_ADD
4591 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)
4592 || (opline->opcode == ZEND_SUB
4593 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) {
4594 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
4595 }
4596 if (!zend_jit_math(&ctx, opline,
4597 op1_info, op1_addr,
4598 op2_info, op2_addr,
4599 res_use_info, res_info, res_addr,
4600 may_overflow,
4601 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4602 goto jit_failure;
4603 }
4604 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4605 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4606 && has_concrete_type(op1_info)
4607 && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
4608 && has_concrete_type(op2_info)
4609 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4610 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4611 }
4612 }
4613 goto done;
4614 case ZEND_CONCAT:
4615 case ZEND_FAST_CONCAT:
4616 op1_info = OP1_INFO();
4617 CHECK_OP1_TRACE_TYPE();
4618 op2_info = OP2_INFO();
4619 CHECK_OP2_TRACE_TYPE();
4620 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4621 break;
4622 }
4623 if (!(op1_info & MAY_BE_STRING) ||
4624 !(op2_info & MAY_BE_STRING)) {
4625 break;
4626 }
4627 res_addr = RES_REG_ADDR();
4628 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4629 send_result = 1;
4630 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4631 if (!zend_jit_reuse_ip(&ctx)) {
4632 goto jit_failure;
4633 }
4634 }
4635 if (!zend_jit_concat(&ctx, opline,
4636 op1_info, op2_info, res_addr,
4637 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4638 goto jit_failure;
4639 }
4640 goto done;
4641 case ZEND_ASSIGN_OP:
4642 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4643 break;
4644 }
4645 op1_info = OP1_INFO();
4646 CHECK_OP1_TRACE_TYPE();
4647 op2_info = OP2_INFO();
4648 CHECK_OP2_TRACE_TYPE();
4649 if (!zend_jit_supported_binary_op(
4650 opline->extended_value, op1_info, op2_info)) {
4651 break;
4652 }
4653 op1_addr = OP1_REG_ADDR();
4654 if (Z_MODE(op1_addr) != IS_REG
4655 || Z_LOAD(op1_addr)
4656 || Z_STORE(op1_addr)) {
4657 op1_mem_info = op1_info;
4658 } else {
4659 op1_mem_info = zend_jit_trace_type_to_info(
4660 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)));
4661 }
4662 op1_def_info = OP1_DEF_INFO();
4663 if (op1_def_info & MAY_BE_GUARD
4664 && !has_concrete_type(op1_def_info)) {
4665 op1_def_info &= ~MAY_BE_GUARD;
4666 }
4667 if (!zend_jit_assign_op(&ctx, opline,
4668 op1_info, op1_addr, OP1_RANGE(),
4669 op1_def_info, OP1_DEF_REG_ADDR(), op1_mem_info,
4670 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
4671 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4672 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4673 goto jit_failure;
4674 }
4675 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4676 && has_concrete_type(op1_info)
4677 && has_concrete_type(op2_info)) {
4678 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4679 if (opline->result_type != IS_UNUSED) {
4680 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4681 }
4682 }
4683 goto done;
4684 case ZEND_ASSIGN_DIM_OP:
4685 if (opline->result_type != IS_UNUSED) {
4686 break;
4687 }
4688 if (!zend_jit_supported_binary_op(
4689 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4690 break;
4691 }
4692 if (opline->op1_type == IS_CV
4693 && (opline+1)->op1_type == IS_CV
4694 && (opline+1)->op1.var == opline->op1.var) {
4695 /* skip $a[x] += $a; */
4696 break;
4697 }
4698 op1_info = OP1_INFO();
4699 op1_addr = OP1_REG_ADDR();
4700 if (opline->op1_type == IS_VAR) {
4701 if (orig_op1_type != IS_UNKNOWN
4702 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4703 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4704 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4705 goto jit_failure;
4706 }
4707 } else {
4708 break;
4709 }
4710 }
4711 if (orig_op1_type != IS_UNKNOWN
4712 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4713 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4714 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4715 goto jit_failure;
4716 }
4717 if (opline->op1_type == IS_CV
4718 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4719 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4720 }
4721 } else {
4722 CHECK_OP1_TRACE_TYPE();
4723 }
4724 op2_info = OP2_INFO();
4725 CHECK_OP2_TRACE_TYPE();
4726 op1_data_info = OP1_DATA_INFO();
4727 CHECK_OP1_DATA_TRACE_TYPE();
4728 op1_def_info = OP1_DEF_INFO();
4729 if (!zend_jit_assign_dim_op(&ctx, opline,
4730 op1_info, op1_def_info, op1_addr,
4731 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
4732 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
4733 op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(), val_type,
4734 zend_jit_trace_may_throw(opline, ssa_op, op_array, ssa,
4735 op1_info, op2_info, op1_data_info, val_type))) {
4736 goto jit_failure;
4737 }
4738 goto done;
4739 case ZEND_PRE_INC_OBJ:
4740 case ZEND_PRE_DEC_OBJ:
4741 case ZEND_POST_INC_OBJ:
4742 case ZEND_POST_DEC_OBJ:
4743 if (opline->op2_type != IS_CONST
4744 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4745 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4746 break;
4747 }
4748 ce = NULL;
4749 ce_is_instanceof = 0;
4750 on_this = delayed_fetch_this = 0;
4751 op1_indirect = 0;
4752 if (opline->op1_type == IS_UNUSED) {
4753 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4754 ce = op_array->scope;
4755 /* scope is NULL for closures. */
4756 if (ce) {
4757 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4758 }
4759 op1_addr = 0;
4760 on_this = 1;
4761 } else {
4762 if (ssa_op->op1_use >= 0) {
4763 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4764 }
4765 op1_info = OP1_INFO();
4766 if (!(op1_info & MAY_BE_OBJECT)) {
4767 break;
4768 }
4769 op1_addr = OP1_REG_ADDR();
4770 if (opline->op1_type == IS_VAR) {
4771 if (orig_op1_type != IS_UNKNOWN
4772 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4773 op1_indirect = 1;
4774 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4775 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4776 goto jit_failure;
4777 }
4778 }
4779 }
4780 if (orig_op1_type != IS_UNKNOWN
4781 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4782 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4783 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4784 goto jit_failure;
4785 }
4786 if (opline->op1_type == IS_CV
4787 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4788 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4789 }
4790 } else {
4791 CHECK_OP1_TRACE_TYPE();
4792 }
4793 if (!(op1_info & MAY_BE_OBJECT)) {
4794 break;
4795 }
4796 if (ssa->var_info && ssa->ops) {
4797 if (ssa_op->op1_use >= 0) {
4798 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4799 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4800 ce = op1_ssa->ce;
4801 ce_is_instanceof = op1_ssa->is_instanceof;
4802 }
4803 }
4804 }
4805 if (delayed_fetch_this) {
4806 on_this = 1;
4807 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4808 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4809 } else if (op_array_ssa->ops
4810 && op_array_ssa->vars
4811 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4812 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4813 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4814 }
4815 }
4816 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
4817 op1_info, op1_addr,
4818 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4819 val_type)) {
4820 goto jit_failure;
4821 }
4822 goto done;
4823 case ZEND_ASSIGN_OBJ_OP:
4824 if (opline->result_type != IS_UNUSED) {
4825 break;
4826 }
4827 if (opline->op2_type != IS_CONST
4828 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4829 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4830 break;
4831 }
4832 if (opline->op1_type == IS_CV
4833 && (opline+1)->op1_type == IS_CV
4834 && (opline+1)->op1.var == opline->op1.var) {
4835 /* skip $a->prop += $a; */
4836 break;
4837 }
4838 if (!zend_jit_supported_binary_op(
4839 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4840 break;
4841 }
4842 ce = NULL;
4843 ce_is_instanceof = 0;
4844 on_this = delayed_fetch_this = 0;
4845 op1_indirect = 0;
4846 if (opline->op1_type == IS_UNUSED) {
4847 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4848 ce = op_array->scope;
4849 /* scope is NULL for closures. */
4850 if (ce) {
4851 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4852 }
4853 op1_addr = 0;
4854 on_this = 1;
4855 } else {
4856 if (ssa_op->op1_use >= 0) {
4857 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4858 }
4859 op1_info = OP1_INFO();
4860 if (!(op1_info & MAY_BE_OBJECT)) {
4861 break;
4862 }
4863 op1_addr = OP1_REG_ADDR();
4864 if (opline->op1_type == IS_VAR) {
4865 if (orig_op1_type != IS_UNKNOWN
4866 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4867 op1_indirect = 1;
4868 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4869 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4870 goto jit_failure;
4871 }
4872 }
4873 }
4874 if (orig_op1_type != IS_UNKNOWN
4875 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4876 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4877 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4878 goto jit_failure;
4879 }
4880 if (opline->op1_type == IS_CV
4881 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4882 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4883 }
4884 } else {
4885 CHECK_OP1_TRACE_TYPE();
4886 }
4887 if (!(op1_info & MAY_BE_OBJECT)) {
4888 break;
4889 }
4890 if (ssa->var_info && ssa->ops) {
4891 if (ssa_op->op1_use >= 0) {
4892 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4893 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4894 ce = op1_ssa->ce;
4895 ce_is_instanceof = op1_ssa->is_instanceof;
4896 }
4897 }
4898 }
4899 if (delayed_fetch_this) {
4900 on_this = 1;
4901 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4902 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4903 } else if (op_array_ssa->ops
4904 && op_array_ssa->vars
4905 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4906 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4907 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4908 }
4909 }
4910 op1_data_info = OP1_DATA_INFO();
4911 CHECK_OP1_DATA_TRACE_TYPE();
4912 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
4913 op1_info, op1_addr, op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(),
4914 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4915 val_type)) {
4916 goto jit_failure;
4917 }
4918 goto done;
4919 case ZEND_ASSIGN_OBJ:
4920 if (opline->op2_type != IS_CONST
4921 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4922 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4923 break;
4924 }
4925 ce = NULL;
4926 ce_is_instanceof = 0;
4927 on_this = delayed_fetch_this = 0;
4928 op1_indirect = 0;
4929 if (opline->op1_type == IS_UNUSED) {
4930 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4931 ce = op_array->scope;
4932 /* scope is NULL for closures. */
4933 if (ce) {
4934 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
4935 }
4936 op1_addr = 0;
4937 on_this = 1;
4938 } else {
4939 if (ssa_op->op1_use >= 0) {
4940 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4941 }
4942 op1_info = OP1_INFO();
4943 if (!(op1_info & MAY_BE_OBJECT)) {
4944 break;
4945 }
4946 op1_addr = OP1_REG_ADDR();
4947 if (opline->op1_type == IS_VAR) {
4948 if (orig_op1_type != IS_UNKNOWN
4949 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4950 op1_indirect = 1;
4951 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4952 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4953 goto jit_failure;
4954 }
4955 }
4956 }
4957 if (orig_op1_type != IS_UNKNOWN
4958 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4959 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4960 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4961 goto jit_failure;
4962 }
4963 if (opline->op1_type == IS_CV
4964 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4965 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4966 }
4967 } else {
4968 CHECK_OP1_TRACE_TYPE();
4969 }
4970 if (!(op1_info & MAY_BE_OBJECT)) {
4971 break;
4972 }
4973 if (ssa->var_info && ssa->ops) {
4974 if (ssa_op->op1_use >= 0) {
4975 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4976 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4977 ce = op1_ssa->ce;
4978 ce_is_instanceof = op1_ssa->is_instanceof;
4979 }
4980 }
4981 }
4982 if (delayed_fetch_this) {
4983 on_this = 1;
4984 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4985 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4986 } else if (op_array_ssa->ops
4987 && op_array_ssa->vars
4988 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4989 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4990 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4991 }
4992 }
4993 op1_data_info = OP1_DATA_INFO();
4994 CHECK_OP1_DATA_TRACE_TYPE();
4995 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
4996 op1_info, op1_addr, op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_DEF_REG_ADDR(),
4997 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
4998 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4999 val_type,
5000 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5001 goto jit_failure;
5002 }
5003 if ((opline+1)->op1_type == IS_CV
5004 && (ssa_op+1)->op1_def >= 0
5005 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
5006 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
5007 }
5008 goto done;
5009 case ZEND_ASSIGN_DIM:
5010 op1_info = OP1_INFO();
5011 op1_addr = OP1_REG_ADDR();
5012 if (opline->op1_type == IS_CV
5013 && (opline+1)->op1_type == IS_CV
5014 && (opline+1)->op1.var == opline->op1.var) {
5015 /* skip $a[x] = $a; */
5016 break;
5017 }
5018 if (opline->op1_type == IS_VAR) {
5019 if (orig_op1_type != IS_UNKNOWN
5020 && (orig_op1_type & IS_TRACE_INDIRECT)
5021 && opline->result_type == IS_UNUSED) {
5022 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
5023 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5024 goto jit_failure;
5025 }
5026 } else {
5027 break;
5028 }
5029 }
5030 if (orig_op1_type != IS_UNKNOWN
5031 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5032 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5033 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5034 goto jit_failure;
5035 }
5036 if (opline->op1_type == IS_CV
5037 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5038 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5039 }
5040 } else {
5041 CHECK_OP1_TRACE_TYPE();
5042 }
5043 op2_info = OP2_INFO();
5044 CHECK_OP2_TRACE_TYPE();
5045 op1_data_info = OP1_DATA_INFO();
5046 CHECK_OP1_DATA_TRACE_TYPE();
5047 if (!zend_jit_assign_dim(&ctx, opline,
5048 op1_info, op1_addr,
5049 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
5050 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
5051 op1_data_info, OP1_DATA_REG_ADDR(),
5052 (ctx.ra && (ssa_op+1)->op1_def >= 0) ? OP1_DATA_DEF_REG_ADDR() : 0,
5053 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
5054 val_type,
5055 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5056 goto jit_failure;
5057 }
5058 if ((opline+1)->op1_type == IS_CV
5059 && (ssa_op+1)->op1_def >= 0
5060 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
5061 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
5062 }
5063 goto done;
5064 case ZEND_ASSIGN:
5065 if (opline->op1_type != IS_CV) {
5066 break;
5067 }
5068 op2_addr = OP2_REG_ADDR();
5069 op2_info = OP2_INFO();
5070 zend_jit_addr ref_addr = 0;
5071
5072 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
5073 op2_def_addr = op2_addr;
5074 } else {
5075 op2_def_addr = OP2_DEF_REG_ADDR();
5076 }
5077 CHECK_OP2_TRACE_TYPE();
5078 op1_info = OP1_INFO();
5079 op1_def_info = OP1_DEF_INFO();
5080 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
5081 if (op1_type < IS_STRING
5082 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
5083 if (!zend_jit_scalar_type_guard(&ctx, opline, opline->op1.var)) {
5084 goto jit_failure;
5085 }
5086 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
5087 } else {
5088 CHECK_OP1_TRACE_TYPE();
5089 }
5090 }
5091 op1_addr = OP1_REG_ADDR();
5092 op1_def_addr = OP1_DEF_REG_ADDR();
5093 if (Z_MODE(op1_def_addr) != IS_REG &&
5094 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
5095 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
5096 /* type may be not set */
5097 op1_info |= MAY_BE_NULL;
5098 }
5099 if (orig_op1_type != IS_UNKNOWN) {
5100 if (orig_op1_type & IS_TRACE_REFERENCE) {
5101 if (!zend_jit_guard_reference(&ctx, opline, &op1_addr, &ref_addr,
5102 !ssa->var_info[ssa_op->op1_use].guarded_reference)) {
5103 goto jit_failure;
5104 }
5105 op1_info &= ~MAY_BE_REF;
5106 if (opline->op1_type == IS_CV
5107 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5108 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5109 }
5110 if (opline->result_type == IS_UNUSED) {
5111 res_addr = 0;
5112 } else {
5113 res_addr = RES_REG_ADDR();
5114 if (Z_MODE(res_addr) != IS_REG
5115 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5116 send_result = 1;
5117 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5118 if (!zend_jit_reuse_ip(&ctx)) {
5119 goto jit_failure;
5120 }
5121 }
5122 }
5123 op1_def_addr = op1_addr;
5124 op1_def_info &= ~MAY_BE_REF;
5125 } else if (op1_info & MAY_BE_REF) {
5126 if (!zend_jit_noref_guard(&ctx, opline, op1_addr)) {
5127 goto jit_failure;
5128 }
5129 op1_info &= ~MAY_BE_REF;
5130 op1_def_info &= ~MAY_BE_REF;
5131 }
5132 }
5133 if (opline->result_type == IS_UNUSED) {
5134 res_addr = 0;
5135 res_info = -1;
5136 } else {
5137 res_addr = RES_REG_ADDR();
5138 res_info = RES_INFO();
5139 if (Z_MODE(res_addr) != IS_REG
5140 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5141 send_result = 1;
5142 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5143 if (!zend_jit_reuse_ip(&ctx)) {
5144 goto jit_failure;
5145 }
5146 }
5147 }
5148 if (!zend_jit_assign(&ctx, opline,
5149 op1_info, op1_addr,
5150 op1_def_info, op1_def_addr,
5151 op2_info, op2_addr, op2_def_addr,
5152 res_info, res_addr,
5153 ref_addr,
5154 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5155 goto jit_failure;
5156 }
5157 if (ssa_op->op2_def >= 0
5158 && Z_MODE(op2_addr) == IS_REG
5159 && ssa->vars[ssa_op->op2_def].no_val) {
5160 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5161 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5162
5163 if (STACK_MEM_TYPE(stack, var_num) != type
5164 && ssa->vars[ssa_op->op2_def].use_chain < 0
5165 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5166 if (!zend_jit_store_type(&ctx, var_num, type)) {
5167 return 0;
5168 }
5169 SET_STACK_TYPE(stack, var_num, type, 1);
5170 }
5171 }
5172 if (opline->op2_type == IS_CV
5173 && ssa_op->op2_def >= 0
5174 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5175 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5176 }
5177 goto done;
5178 case ZEND_CAST:
5179 if (opline->extended_value != op1_type) {
5180 break;
5181 }
5182 ZEND_FALLTHROUGH;
5183 case ZEND_QM_ASSIGN:
5184 op1_addr = OP1_REG_ADDR();
5185 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5186 op1_def_addr = op1_addr;
5187 } else {
5188 op1_def_addr = OP1_DEF_REG_ADDR();
5189 }
5190 op1_info = OP1_INFO();
5191 CHECK_OP1_TRACE_TYPE();
5192 res_info = RES_INFO();
5193 res_use_info = zend_jit_trace_type_to_info(
5194 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
5195 if (opline->result_type == IS_CV) {
5196 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
5197 }
5198 res_addr = RES_REG_ADDR();
5199 if (Z_MODE(res_addr) != IS_REG &&
5200 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5201 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5202 /* type may be not set */
5203 res_use_info |= MAY_BE_NULL;
5204 }
5205 if (!zend_jit_qm_assign(&ctx, opline,
5206 op1_info, op1_addr, op1_def_addr,
5207 res_use_info, res_info, res_addr)) {
5208 goto jit_failure;
5209 }
5210 if (ssa_op->op1_def >= 0
5211 && Z_MODE(op1_addr) == IS_REG
5212 && ssa->vars[ssa_op->op1_def].no_val) {
5213 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5214 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5215
5216 if (STACK_MEM_TYPE(stack, var_num) != type
5217 && ssa->vars[ssa_op->op1_def].use_chain < 0
5218 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5219 if (!zend_jit_store_type(&ctx, var_num, type)) {
5220 return 0;
5221 }
5222 SET_STACK_TYPE(stack, var_num, type, 1);
5223 }
5224 }
5225 if (opline->op1_type == IS_CV
5226 && ssa_op->op1_def >= 0
5227 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5228 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5229 }
5230 goto done;
5231 case ZEND_INIT_FCALL:
5232 case ZEND_INIT_FCALL_BY_NAME:
5233 case ZEND_INIT_NS_FCALL_BY_NAME:
5234 frame_flags = TRACE_FRAME_MASK_NESTED;
5235 if (!zend_jit_init_fcall(&ctx, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5236 goto jit_failure;
5237 }
5238 goto done;
5239 case ZEND_SEND_VAL:
5240 case ZEND_SEND_VAL_EX:
5241 if (opline->op2_type == IS_CONST) {
5242 /* Named parameters not supported in JIT */
5243 break;
5244 }
5245 if (opline->opcode == ZEND_SEND_VAL_EX
5246 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5247 break;
5248 }
5249 op1_info = OP1_INFO();
5250 CHECK_OP1_TRACE_TYPE();
5251 if (!zend_jit_send_val(&ctx, opline,
5252 op1_info, OP1_REG_ADDR())) {
5253 goto jit_failure;
5254 }
5255 if (frame->call && frame->call->func) {
5256 if (opline->op1_type == IS_CONST) {
5257 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5258 } else if (op1_type != IS_UNKNOWN) {
5259 if (op1_type == IS_UNDEF) {
5260 op1_type = IS_NULL;
5261 }
5262 zend_jit_trace_send_type(opline, frame->call, op1_type);
5263 }
5264 }
5265 goto done;
5266 case ZEND_SEND_REF:
5267 if (opline->op2_type == IS_CONST) {
5268 /* Named parameters not supported in JIT */
5269 break;
5270 }
5271 op1_info = OP1_INFO();
5272 if (!zend_jit_send_ref(&ctx, opline, op_array,
5273 op1_info, 0)) {
5274 goto jit_failure;
5275 }
5276 if (opline->op1_type == IS_CV
5277 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5278 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5279 }
5280 goto done;
5281 case ZEND_SEND_VAR:
5282 case ZEND_SEND_VAR_EX:
5283 case ZEND_SEND_VAR_NO_REF:
5284 case ZEND_SEND_VAR_NO_REF_EX:
5285 case ZEND_SEND_FUNC_ARG:
5286 if (opline->op2_type == IS_CONST) {
5287 /* Named parameters not supported in JIT */
5288 break;
5289 }
5290 if ((opline->opcode == ZEND_SEND_VAR_EX
5291 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5292 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5293 break;
5294 }
5295 op1_addr = OP1_REG_ADDR();
5296 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5297 op1_def_addr = op1_addr;
5298 } else {
5299 op1_def_addr = OP1_DEF_REG_ADDR();
5300 }
5301 op1_info = OP1_INFO();
5302 CHECK_OP1_TRACE_TYPE();
5303 if (!zend_jit_send_var(&ctx, opline, op_array,
5304 op1_info, op1_addr, op1_def_addr)) {
5305 goto jit_failure;
5306 }
5307 if (ssa_op->op1_def >= 0
5308 && Z_MODE(op1_addr) == IS_REG
5309 && ssa->vars[ssa_op->op1_def].no_val) {
5310 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5311 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5312
5313 if (STACK_MEM_TYPE(stack, var_num) != type
5314 && ssa->vars[ssa_op->op1_def].use_chain < 0
5315 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5316 if (!zend_jit_store_type(&ctx, var_num, type)) {
5317 return 0;
5318 }
5319 SET_STACK_TYPE(stack, var_num, type, 1);
5320 }
5321 }
5322 if (opline->op1_type == IS_CV
5323 && ssa_op->op1_def >= 0
5324 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5325 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5326 }
5327 if (frame->call && frame->call->func) {
5328 if ((opline->opcode == ZEND_SEND_VAR_EX
5329 || opline->opcode == ZEND_SEND_FUNC_ARG)
5330 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5331 goto done;
5332 }
5333 if (op1_type != IS_UNKNOWN) {
5334 if (op1_type == IS_UNDEF) {
5335 op1_type = IS_NULL;
5336 }
5337 zend_jit_trace_send_type(opline, frame->call, op1_type);
5338 }
5339 }
5340 goto done;
5341 case ZEND_CHECK_FUNC_ARG:
5342 if (!JIT_G(current_frame)
5343 || !JIT_G(current_frame)->call
5344 || !JIT_G(current_frame)->call->func) {
5345 break;
5346 }
5347 if (opline->op2_type == IS_CONST
5348 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5349 /* Named parameters not supported in JIT */
5350 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5351 break;
5352 }
5353 if (!zend_jit_check_func_arg(&ctx, opline)) {
5354 goto jit_failure;
5355 }
5356 goto done;
5357 case ZEND_CHECK_UNDEF_ARGS:
5358 if (JIT_G(current_frame)
5359 && JIT_G(current_frame)->call) {
5360 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5361 }
5362 if (!zend_jit_check_undef_args(&ctx, opline)) {
5363 goto jit_failure;
5364 }
5365 goto done;
5366 case ZEND_DO_UCALL:
5367 case ZEND_DO_ICALL:
5368 case ZEND_DO_FCALL_BY_NAME:
5369 case ZEND_DO_FCALL:
5370 if (!zend_jit_do_fcall(&ctx, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5371 goto jit_failure;
5372 }
5373 goto done;
5374 case ZEND_IS_EQUAL:
5375 case ZEND_IS_NOT_EQUAL:
5376 case ZEND_IS_SMALLER:
5377 case ZEND_IS_SMALLER_OR_EQUAL:
5378 case ZEND_CASE:
5379 op1_info = OP1_INFO();
5380 op2_info = OP2_INFO();
5381 skip_comparison =
5382 ssa_op != ssa->ops &&
5383 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5384 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5385 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5386 CHECK_OP1_TRACE_TYPE();
5387 CHECK_OP2_TRACE_TYPE();
5388 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5389 bool exit_if_true = 0;
5390 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5391 uint32_t exit_point;
5392
5393 if (ra) {
5394 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5395 }
5396 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5397 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5398 if (!exit_addr) {
5399 goto jit_failure;
5400 }
5401 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5402 if (!zend_jit_cmp(&ctx, opline,
5403 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5404 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5405 RES_REG_ADDR(),
5406 zend_may_throw(opline, ssa_op, op_array, ssa),
5407 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5408 goto jit_failure;
5409 }
5410 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5411 } else {
5412 smart_branch_opcode = 0;
5413 exit_addr = NULL;
5414 if (!zend_jit_cmp(&ctx, opline,
5415 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5416 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5417 RES_REG_ADDR(),
5418 zend_may_throw(opline, ssa_op, op_array, ssa),
5419 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5420 goto jit_failure;
5421 }
5422 }
5423 goto done;
5424 case ZEND_IS_IDENTICAL:
5425 case ZEND_IS_NOT_IDENTICAL:
5426 case ZEND_CASE_STRICT:
5427 op1_info = OP1_INFO();
5428 op2_info = OP2_INFO();
5429 skip_comparison =
5430 ssa_op != ssa->ops &&
5431 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5432 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5433 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5434 CHECK_OP1_TRACE_TYPE();
5435 CHECK_OP2_TRACE_TYPE();
5436 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5437 bool exit_if_true = 0;
5438 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5439 uint32_t exit_point;
5440
5441 if (ra) {
5442 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5443 }
5444 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5445 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5446 if (!exit_addr) {
5447 goto jit_failure;
5448 }
5449 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5450 exit_if_true = !exit_if_true;
5451 }
5452 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5453 if (!zend_jit_identical(&ctx, opline,
5454 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5455 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5456 RES_REG_ADDR(),
5457 zend_may_throw(opline, ssa_op, op_array, ssa),
5458 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5459 goto jit_failure;
5460 }
5461 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5462 } else {
5463 smart_branch_opcode = 0;
5464 exit_addr = NULL;
5465 if (!zend_jit_identical(&ctx, opline,
5466 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5467 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5468 RES_REG_ADDR(),
5469 zend_may_throw(opline, ssa_op, op_array, ssa),
5470 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5471 goto jit_failure;
5472 }
5473 }
5474 goto done;
5475 case ZEND_DEFINED:
5476 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5477 bool exit_if_true = 0;
5478 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5479 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5480
5481 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5482 if (!exit_addr) {
5483 goto jit_failure;
5484 }
5485 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5486 } else {
5487 smart_branch_opcode = 0;
5488 exit_addr = NULL;
5489 }
5490 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5491 goto jit_failure;
5492 }
5493 goto done;
5494 case ZEND_TYPE_CHECK:
5495 if (opline->extended_value == MAY_BE_RESOURCE) {
5496 // TODO: support for is_resource() ???
5497 break;
5498 }
5499 op1_info = OP1_INFO();
5500 CHECK_OP1_TRACE_TYPE();
5501 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5502 bool exit_if_true = 0;
5503 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5504 uint32_t exit_point;
5505
5506 if (ra) {
5507 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5508 }
5509 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5510 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5511 if (!exit_addr) {
5512 goto jit_failure;
5513 }
5514 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5515 } else {
5516 smart_branch_opcode = 0;
5517 exit_addr = NULL;
5518 }
5519 if (!zend_jit_type_check(&ctx, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5520 goto jit_failure;
5521 }
5522 goto done;
5523 case ZEND_RETURN:
5524 op1_info = OP1_INFO();
5525 CHECK_OP1_TRACE_TYPE();
5526 if (opline->op1_type == IS_CONST) {
5527 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5528 } else if (op1_type != IS_UNKNOWN) {
5529 res_type = op1_type;
5530 if (res_type == IS_UNDEF) {
5531 res_type = IS_NULL;
5532 }
5533 }
5534 if (op_array->type == ZEND_EVAL_CODE
5535 // TODO: support for top-level code
5536 || !op_array->function_name
5537 // TODO: support for IS_UNDEF ???
5538 || (op1_info & MAY_BE_UNDEF)) {
5539 if (!zend_jit_trace_handler(&ctx, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5540 goto jit_failure;
5541 }
5542 } else {
5543 int j;
5544 int may_throw = 0;
5545 bool left_frame = 0;
5546
5547 if (!zend_jit_return(&ctx, opline, op_array,
5548 op1_info, OP1_REG_ADDR())) {
5549 goto jit_failure;
5550 }
5551 if (op_array->last_var > 100) {
5552 /* To many CVs to unroll */
5553 if (!zend_jit_free_cvs(&ctx)) {
5554 goto jit_failure;
5555 }
5556 left_frame = 1;
5557 }
5558 if (!left_frame) {
5559 for (j = 0 ; j < op_array->last_var; j++) {
5560 uint32_t info;
5561 uint8_t type;
5562
5563 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5564 type = STACK_TYPE(stack, j);
5565 info = zend_jit_trace_type_to_info_ex(type, info);
5566 if (opline->op1_type == IS_CV
5567 && EX_VAR_TO_NUM(opline->op1.var) == j
5568 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5569 if (JIT_G(current_frame)
5570 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5571 continue;
5572 } else {
5573 info |= MAY_BE_NULL;
5574 }
5575 }
5576 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5577 if (!left_frame) {
5578 left_frame = 1;
5579 if (!zend_jit_leave_frame(&ctx)) {
5580 goto jit_failure;
5581 }
5582 }
5583 if (!zend_jit_free_cv(&ctx, info, j)) {
5584 goto jit_failure;
5585 }
5586 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5587 if (info & MAY_BE_RC1) {
5588 may_throw = 1;
5589 }
5590 }
5591 }
5592 }
5593 }
5594 if (!zend_jit_leave_func(&ctx, op_array, opline, op1_info, left_frame,
5595 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5596 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5597 goto jit_failure;
5598 }
5599 }
5600 goto done;
5601 case ZEND_BOOL:
5602 case ZEND_BOOL_NOT:
5603 op1_info = OP1_INFO();
5604 CHECK_OP1_TRACE_TYPE();
5605 if (!zend_jit_bool_jmpznz(&ctx, opline,
5606 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5607 -1, -1,
5608 zend_may_throw(opline, ssa_op, op_array, ssa),
5609 opline->opcode, NULL)) {
5610 goto jit_failure;
5611 }
5612 goto done;
5613 case ZEND_JMPZ:
5614 case ZEND_JMPNZ:
5615 case ZEND_JMPZ_EX:
5616 case ZEND_JMPNZ_EX:
5617 op1_info = OP1_INFO();
5618 CHECK_OP1_TRACE_TYPE();
5619 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5620 const zend_op *exit_opline = NULL;
5621 uint32_t exit_point;
5622
5623 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5624 /* taken branch */
5625 if (opline->opcode == ZEND_JMPNZ_EX) {
5626 smart_branch_opcode = ZEND_JMPZ_EX;
5627 } else if (opline->opcode == ZEND_JMPZ_EX) {
5628 smart_branch_opcode = ZEND_JMPNZ_EX;
5629 } else if (opline->opcode == ZEND_JMPNZ) {
5630 smart_branch_opcode = ZEND_JMPZ;
5631 } else {
5632 smart_branch_opcode = ZEND_JMPNZ;
5633 }
5634 exit_opline = opline + 1;
5635 } else if ((p+1)->opline == opline + 1) {
5636 /* not taken branch */
5637 smart_branch_opcode = opline->opcode;
5638 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5639 } else {
5640 ZEND_UNREACHABLE();
5641 }
5642 if (ra) {
5643 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5644 }
5645 if (!(op1_info & MAY_BE_GUARD)
5646 && has_concrete_type(op1_info)
5647 && concrete_type(op1_info) <= IS_TRUE) {
5648 /* unconditional branch */
5649 exit_addr = NULL;
5650 } else if (opline->result_type == IS_TMP_VAR) {
5651 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5652
5653 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5654 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5655 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5656 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5657 if (!exit_addr) {
5658 goto jit_failure;
5659 }
5660 } else {
5661 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5662 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5663 if (!exit_addr) {
5664 goto jit_failure;
5665 }
5666 }
5667 } else {
5668 ZEND_UNREACHABLE();
5669 }
5670 if (opline->result_type == IS_UNDEF) {
5671 res_addr = 0;
5672 } else {
5673 res_addr = RES_REG_ADDR();
5674 }
5675 if (!zend_jit_bool_jmpznz(&ctx, opline,
5676 op1_info, OP1_REG_ADDR(), res_addr,
5677 -1, -1,
5678 zend_may_throw(opline, ssa_op, op_array, ssa),
5679 smart_branch_opcode, exit_addr)) {
5680 goto jit_failure;
5681 }
5682 goto done;
5683 case ZEND_JMP_FRAMELESS:
5684 op1_info = OP1_INFO();
5685 ZEND_ASSERT((p+1)->op == ZEND_JIT_TRACE_VM);
5686 const zend_op *exit_opline = NULL;
5687 uint32_t exit_point;
5688 zend_jmp_fl_result guard;
5689
5690 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5691 /* taken branch */
5692 guard = ZEND_JMP_FL_HIT;
5693 exit_opline = opline + 1;
5694 } else if ((p+1)->opline == opline + 1) {
5695 /* not taken branch */
5696 guard = ZEND_JMP_FL_MISS;
5697 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5698 } else {
5699 ZEND_UNREACHABLE();
5700 }
5701 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5702 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5703 if (!exit_addr) {
5704 goto jit_failure;
5705 }
5706 if (!zend_jit_jmp_frameless(&ctx, opline, exit_addr, guard)) {
5707 goto jit_failure;
5708 }
5709 goto done;
5710 case ZEND_ISSET_ISEMPTY_CV:
5711 if ((opline->extended_value & ZEND_ISEMPTY)) {
5712 // TODO: support for empty() ???
5713 break;
5714 }
5715 op1_info = OP1_INFO();
5716 op1_addr = OP1_REG_ADDR();
5717 if (orig_op1_type != IS_UNKNOWN
5718 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5719 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5720 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5721 goto jit_failure;
5722 }
5723 if (opline->op1_type == IS_CV
5724 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5725 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5726 }
5727 } else {
5728 CHECK_OP1_TRACE_TYPE();
5729 }
5730 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5731 bool exit_if_true = 0;
5732 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5733 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5734
5735 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5736 if (!exit_addr) {
5737 goto jit_failure;
5738 }
5739 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5740 } else {
5741 smart_branch_opcode = 0;
5742 exit_addr = NULL;
5743 }
5744 if (!zend_jit_isset_isempty_cv(&ctx, opline,
5745 op1_info, op1_addr,
5746 smart_branch_opcode, -1, -1, exit_addr)) {
5747 goto jit_failure;
5748 }
5749 goto done;
5750 case ZEND_IN_ARRAY:
5751 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5752 break;
5753 }
5754 op1_info = OP1_INFO();
5755 op1_addr = OP1_REG_ADDR();
5756 CHECK_OP1_TRACE_TYPE();
5757 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5758 break;
5759 }
5760 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5761 bool exit_if_true = 0;
5762 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5763 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5764
5765 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5766 if (!exit_addr) {
5767 goto jit_failure;
5768 }
5769 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5770 } else {
5771 smart_branch_opcode = 0;
5772 exit_addr = NULL;
5773 }
5774 if (!zend_jit_in_array(&ctx, opline,
5775 op1_info, op1_addr,
5776 smart_branch_opcode, -1, -1, exit_addr)) {
5777 goto jit_failure;
5778 }
5779 goto done;
5780 case ZEND_FETCH_DIM_FUNC_ARG:
5781 if (!JIT_G(current_frame)
5782 || !JIT_G(current_frame)->call
5783 || !JIT_G(current_frame)->call->func
5784 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5785 break;
5786 }
5787 ZEND_FALLTHROUGH;
5788 case ZEND_FETCH_DIM_R:
5789 case ZEND_FETCH_DIM_IS:
5790 case ZEND_FETCH_LIST_R:
5791 op1_info = OP1_INFO();
5792 op1_addr = OP1_REG_ADDR();
5793 if (orig_op1_type != IS_UNKNOWN
5794 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5795 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5796 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5797 goto jit_failure;
5798 }
5799 if (opline->op1_type == IS_CV
5800 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5801 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5802 if (ssa_op->op1_def >= 0) {
5803 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5804 }
5805 }
5806 } else {
5807 CHECK_OP1_TRACE_TYPE();
5808 }
5809 op2_info = OP2_INFO();
5810 CHECK_OP2_TRACE_TYPE();
5811 res_info = RES_INFO();
5812 avoid_refcounting =
5813 ssa_op->op1_use >= 0 &&
5814 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5815 if (op1_info & MAY_BE_PACKED_GUARD) {
5816 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5817 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5818 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5819 && MAY_BE_PACKED(op1_info)
5820 && MAY_BE_HASH(op1_info)
5821 && orig_op1_type != IS_UNKNOWN) {
5822 op1_info |= MAY_BE_PACKED_GUARD;
5823 if (orig_op1_type & IS_TRACE_PACKED) {
5824 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5825 if (op1_type != IS_UNKNOWN) {
5826 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5827 }
5828 } else {
5829 op1_info &= ~MAY_BE_ARRAY_PACKED;
5830 if (op1_type != IS_UNKNOWN) {
5831 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5832 }
5833 }
5834 }
5835 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
5836 op1_info, op1_addr, avoid_refcounting,
5837 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
5838 res_info, RES_REG_ADDR(), val_type)) {
5839 goto jit_failure;
5840 }
5841 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5842 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5843 }
5844 goto done;
5845 case ZEND_FETCH_DIM_W:
5846 case ZEND_FETCH_DIM_RW:
5847 // case ZEND_FETCH_DIM_UNSET:
5848 case ZEND_FETCH_LIST_W:
5849 if (opline->op1_type != IS_CV
5850 && (orig_op1_type == IS_UNKNOWN
5851 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5852 break;
5853 }
5854 op1_info = OP1_INFO();
5855 op1_addr = OP1_REG_ADDR();
5856 if (opline->op1_type == IS_VAR) {
5857 if (orig_op1_type != IS_UNKNOWN
5858 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5859 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
5860 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5861 goto jit_failure;
5862 }
5863 } else {
5864 break;
5865 }
5866 }
5867 if (orig_op1_type != IS_UNKNOWN
5868 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5869 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5870 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5871 goto jit_failure;
5872 }
5873 if (opline->op1_type == IS_CV
5874 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5875 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5876 }
5877 } else {
5878 CHECK_OP1_TRACE_TYPE();
5879 }
5880 op2_info = OP2_INFO();
5881 CHECK_OP2_TRACE_TYPE();
5882 op1_def_info = OP1_DEF_INFO();
5883 if (!zend_jit_fetch_dim(&ctx, opline,
5884 op1_info, op1_addr,
5885 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
5886 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
5887 RES_REG_ADDR(), val_type)) {
5888 goto jit_failure;
5889 }
5890 if (ssa_op->result_def >= 0
5891 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5892 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5893 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5894 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5895 }
5896 goto done;
5897 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5898 if ((opline->extended_value & ZEND_ISEMPTY)) {
5899 // TODO: support for empty() ???
5900 break;
5901 }
5902 op1_info = OP1_INFO();
5903 op1_addr = OP1_REG_ADDR();
5904 if (orig_op1_type != IS_UNKNOWN
5905 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5906 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5907 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5908 goto jit_failure;
5909 }
5910 if (opline->op1_type == IS_CV
5911 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5912 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5913 }
5914 } else {
5915 CHECK_OP1_TRACE_TYPE();
5916 }
5917 op2_info = OP2_INFO();
5918 CHECK_OP2_TRACE_TYPE();
5919 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5920 bool exit_if_true = 0;
5921 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5922 uint32_t exit_point;
5923 int32_t old_ref = 0;
5924 uint8_t old_flags = 0;
5925
5926 if (ra) {
5927 if (opline->op2_type != IS_CONST) {
5928 old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->op2.var));
5929 old_flags = STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op2.var));
5930 }
5931 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5932 }
5933 if (ssa_op->op1_use >= 0
5934 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5935 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5936 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5937
5938 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5939 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5940 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5941 } else {
5942 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5943 }
5944 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5945 if (!exit_addr) {
5946 goto jit_failure;
5947 }
5948 if (old_ref) {
5949 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op2.var), old_ref, old_flags);
5950 }
5951 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5952 } else {
5953 smart_branch_opcode = 0;
5954 exit_addr = NULL;
5955 }
5956 avoid_refcounting =
5957 ssa_op->op1_use >= 0 &&
5958 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5959 if (op1_info & MAY_BE_PACKED_GUARD) {
5960 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5961 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5962 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5963 && MAY_BE_PACKED(op1_info)
5964 && MAY_BE_HASH(op1_info)
5965 && orig_op1_type != IS_UNKNOWN) {
5966 op1_info |= MAY_BE_PACKED_GUARD;
5967 if (orig_op1_type & IS_TRACE_PACKED) {
5968 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5969 } else {
5970 op1_info &= ~MAY_BE_ARRAY_PACKED;
5971 }
5972 }
5973 if (!zend_jit_isset_isempty_dim(&ctx, opline,
5974 op1_info, op1_addr, avoid_refcounting,
5975 op2_info, OP2_REG_ADDR(), OP2_RANGE(), val_type,
5976 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5977 smart_branch_opcode, -1, -1,
5978 exit_addr)) {
5979 goto jit_failure;
5980 }
5981 goto done;
5982 case ZEND_FETCH_OBJ_FUNC_ARG:
5983 if (!JIT_G(current_frame)
5984 || !JIT_G(current_frame)->call
5985 || !JIT_G(current_frame)->call->func
5986 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5987 break;
5988 }
5989 ZEND_FALLTHROUGH;
5990 case ZEND_FETCH_OBJ_R:
5991 case ZEND_FETCH_OBJ_IS:
5992 case ZEND_FETCH_OBJ_W:
5993 on_this = delayed_fetch_this = 0;
5994 avoid_refcounting = 0;
5995 if (opline->op2_type != IS_CONST
5996 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5997 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5998 break;
5999 }
6000 ce = NULL;
6001 ce_is_instanceof = 0;
6002 op1_indirect = 0;
6003 if (opline->op1_type == IS_UNUSED) {
6004 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6005 ce = op_array->scope;
6006 /* scope is NULL for closures. */
6007 if (ce) {
6008 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
6009 }
6010 op1_addr = 0;
6011 on_this = 1;
6012 } else {
6013 op1_info = OP1_INFO();
6014 if (!(op1_info & MAY_BE_OBJECT)) {
6015 break;
6016 }
6017 op1_addr = OP1_REG_ADDR();
6018 if (opline->op1_type == IS_VAR
6019 && opline->opcode == ZEND_FETCH_OBJ_W) {
6020 if (orig_op1_type != IS_UNKNOWN
6021 && (orig_op1_type & IS_TRACE_INDIRECT)) {
6022 op1_indirect = 1;
6023 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
6024 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
6025 goto jit_failure;
6026 }
6027 }
6028 }
6029 if (orig_op1_type != IS_UNKNOWN
6030 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6031 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6032 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6033 goto jit_failure;
6034 }
6035 if (opline->op1_type == IS_CV
6036 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6037 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
6038 }
6039 } else {
6040 CHECK_OP1_TRACE_TYPE();
6041 }
6042 if (!(op1_info & MAY_BE_OBJECT)) {
6043 break;
6044 }
6045 if (ssa->var_info && ssa->ops) {
6046 if (ssa_op->op1_use >= 0) {
6047 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6048 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6049 ce = op1_ssa->ce;
6050 ce_is_instanceof = op1_ssa->is_instanceof;
6051 }
6052 }
6053 }
6054 if (ssa_op->op1_use >= 0) {
6055 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6056 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
6057 }
6058 if (delayed_fetch_this) {
6059 on_this = 1;
6060 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6061 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6062 } else if (op_array_ssa->ops
6063 && op_array_ssa->vars
6064 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6065 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6066 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6067 }
6068 }
6069 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
6070 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
6071 on_this, delayed_fetch_this, avoid_refcounting, op1_ce,
6072 RES_REG_ADDR(), val_type,
6073 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
6074 goto jit_failure;
6075 }
6076 goto done;
6077 case ZEND_BIND_GLOBAL:
6078 orig_opline = opline;
6079 orig_ssa_op = ssa_op;
6080 while (1) {
6081 if (!ssa->ops || !ssa->var_info) {
6082 op1_info = MAY_BE_ANY|MAY_BE_REF;
6083 } else {
6084 op1_info = OP1_INFO();
6085 }
6086 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
6087 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
6088 }
6089 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
6090 goto jit_failure;
6091 }
6092 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
6093 opline++;
6094 ssa_op++;
6095 } else {
6096 break;
6097 }
6098 }
6099 opline = orig_opline;
6100 ssa_op = orig_ssa_op;
6101 goto done;
6102 case ZEND_RECV:
6103 if (!zend_jit_recv(&ctx, opline, op_array)) {
6104 goto jit_failure;
6105 }
6106 goto done;
6107 case ZEND_RECV_INIT:
6108 orig_opline = opline;
6109 orig_ssa_op = ssa_op;
6110 while (1) {
6111 if (!zend_jit_recv_init(&ctx, opline, op_array,
6112 (opline + 1)->opcode != ZEND_RECV_INIT,
6113 zend_may_throw(opline, ssa_op, op_array, ssa))) {
6114 goto jit_failure;
6115 }
6116 if ((opline+1)->opcode == ZEND_RECV_INIT) {
6117 opline++;
6118 ssa_op++;
6119 } else {
6120 break;
6121 }
6122 }
6123 opline = orig_opline;
6124 ssa_op = orig_ssa_op;
6125 goto done;
6126 case ZEND_FREE:
6127 case ZEND_FE_FREE:
6128 op1_info = OP1_INFO();
6129 if (!zend_jit_free(&ctx, opline, op1_info,
6130 zend_may_throw(opline, ssa_op, op_array, ssa))) {
6131 goto jit_failure;
6132 }
6133 goto done;
6134 case ZEND_ECHO:
6135 op1_info = OP1_INFO();
6136 CHECK_OP1_TRACE_TYPE();
6137 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6138 break;
6139 }
6140 if (!zend_jit_echo(&ctx, opline, op1_info)) {
6141 goto jit_failure;
6142 }
6143 goto done;
6144 case ZEND_STRLEN:
6145 op1_info = OP1_INFO();
6146 op1_addr = OP1_REG_ADDR();
6147 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
6148 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6149 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6150 goto jit_failure;
6151 }
6152 if (opline->op1_type == IS_CV
6153 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6154 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6155 }
6156 } else {
6157 CHECK_OP1_TRACE_TYPE();
6158 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6159 break;
6160 }
6161 }
6162 if (!zend_jit_strlen(&ctx, opline, op1_info, op1_addr, RES_REG_ADDR())) {
6163 goto jit_failure;
6164 }
6165 goto done;
6166 case ZEND_COUNT:
6167 op1_info = OP1_INFO();
6168 op1_addr = OP1_REG_ADDR();
6169 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
6170 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6171 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6172 goto jit_failure;
6173 }
6174 if (opline->op1_type == IS_CV
6175 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6176 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6177 }
6178 } else {
6179 CHECK_OP1_TRACE_TYPE();
6180 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6181 break;
6182 }
6183 }
6184 if (!zend_jit_count(&ctx, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6185 goto jit_failure;
6186 }
6187 goto done;
6188 case ZEND_FETCH_THIS:
6189 delayed_fetch_this = 0;
6190 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6191 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6192 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6193 delayed_fetch_this = 1;
6194 }
6195 }
6196 if (!zend_jit_fetch_this(&ctx, opline, op_array, delayed_fetch_this)) {
6197 goto jit_failure;
6198 }
6199 goto done;
6200 case ZEND_SWITCH_LONG:
6201 case ZEND_SWITCH_STRING:
6202 case ZEND_MATCH:
6203 if (!zend_jit_switch(&ctx, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6204 goto jit_failure;
6205 }
6206 goto done;
6207 case ZEND_VERIFY_RETURN_TYPE:
6208 if (opline->op1_type == IS_UNUSED) {
6209 /* Always throws */
6210 break;
6211 }
6212 if (opline->op1_type == IS_CONST) {
6213 /* TODO Different instruction format, has return value */
6214 break;
6215 }
6216 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6217 /* Not worth bothering with */
6218 break;
6219 }
6220 op1_info = OP1_INFO();
6221 CHECK_OP1_TRACE_TYPE();
6222 if (op1_info & MAY_BE_REF) {
6223 /* TODO May need reference unwrapping. */
6224 break;
6225 }
6226 if (!zend_jit_verify_return_type(&ctx, opline, op_array, op1_info)) {
6227 goto jit_failure;
6228 }
6229 goto done;
6230 case ZEND_FE_RESET_R:
6231 op1_info = OP1_INFO();
6232 CHECK_OP1_TRACE_TYPE();
6233 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6234 break;
6235 }
6236 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
6237 goto jit_failure;
6238 }
6239 goto done;
6240 case ZEND_FE_FETCH_R:
6241 op1_info = OP1_INFO();
6242 CHECK_OP1_TRACE_TYPE();
6243 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6244 break;
6245 }
6246 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6247 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6248 uint32_t exit_point;
6249
6250 if ((p+1)->opline == exit_opline) {
6251 /* taken branch (exit from loop) */
6252 exit_opline = opline;
6253 smart_branch_opcode = ZEND_NOP;
6254 } else if ((p+1)->opline == opline + 1) {
6255 /* not taken branch (loop) */
6256 smart_branch_opcode = ZEND_JMP;
6257 } else {
6258 ZEND_UNREACHABLE();
6259 }
6260 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6261 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6262 if (!exit_addr) {
6263 goto jit_failure;
6264 }
6265 } else {
6266 ZEND_UNREACHABLE();
6267 }
6268 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
6269 -1, smart_branch_opcode, exit_addr)) {
6270 goto jit_failure;
6271 }
6272 goto done;
6273 case ZEND_FETCH_CONSTANT:
6274 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6275 goto jit_failure;
6276 }
6277 goto done;
6278 case ZEND_INIT_METHOD_CALL:
6279 if (opline->op2_type != IS_CONST
6280 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6281 break;
6282 }
6283 on_this = delayed_fetch_this = 0;
6284 ce = NULL;
6285 ce_is_instanceof = 0;
6286 if (opline->op1_type == IS_UNUSED) {
6287 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6288 ce = op_array->scope;
6289 /* scope is NULL for closures. */
6290 if (ce) {
6291 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
6292 }
6293 op1_addr = 0;
6294 on_this = 1;
6295 } else {
6296 op1_info = OP1_INFO();
6297 op1_addr = OP1_REG_ADDR();
6298 if (polymorphic_side_trace) {
6299 op1_info = MAY_BE_OBJECT;
6300 } else if (orig_op1_type != IS_UNKNOWN
6301 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6302 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6303 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6304 goto jit_failure;
6305 }
6306 if (opline->op1_type == IS_CV
6307 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6308 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6309 }
6310 } else {
6311 CHECK_OP1_TRACE_TYPE();
6312 }
6313 if (ssa->var_info && ssa->ops) {
6314 if (ssa_op->op1_use >= 0) {
6315 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6316 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6317 ce = op1_ssa->ce;
6318 ce_is_instanceof = op1_ssa->is_instanceof;
6319 }
6320 }
6321 }
6322 if (ssa_op->op1_use >= 0) {
6323 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6324 }
6325 if (delayed_fetch_this) {
6326 on_this = 1;
6327 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6328 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6329 } else if (op_array_ssa->ops
6330 && op_array_ssa->vars
6331 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6332 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6333 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6334 }
6335 }
6336 frame_flags = TRACE_FRAME_MASK_NESTED;
6337 if (!zend_jit_init_method_call(&ctx, opline,
6338 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6339 op_array, ssa, ssa_op, frame->call_level,
6340 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6341 p + 1, peek_checked_stack - checked_stack,
6342 polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg : -1,
6343 polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg : -1,
6344 polymorphic_side_trace)) {
6345 goto jit_failure;
6346 }
6347 goto done;
6348 case ZEND_INIT_DYNAMIC_CALL:
6349 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6350 break;
6351 }
6352 op2_info = OP2_INFO();
6353 CHECK_OP2_TRACE_TYPE();
6354 frame_flags = TRACE_FRAME_MASK_NESTED;
6355 if (!zend_jit_init_closure_call(&ctx, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6356 goto jit_failure;
6357 }
6358 goto done;
6359 case ZEND_SEND_ARRAY:
6360 case ZEND_SEND_UNPACK:
6361 if (JIT_G(current_frame)
6362 && JIT_G(current_frame)->call) {
6363 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6364 }
6365 break;
6366 case ZEND_ROPE_INIT:
6367 case ZEND_ROPE_ADD:
6368 case ZEND_ROPE_END:
6369 op2_info = OP2_INFO();
6370 CHECK_OP2_TRACE_TYPE();
6371 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6372 break;
6373 }
6374 if (!zend_jit_rope(&ctx, opline, op2_info)) {
6375 goto jit_failure;
6376 }
6377 goto done;
6378 case ZEND_FRAMELESS_ICALL_0:
6379 jit_frameless_icall0(jit, opline);
6380 goto done;
6381 case ZEND_FRAMELESS_ICALL_1:
6382 op1_info = OP1_INFO();
6383 jit_frameless_icall1(jit, opline, op1_info);
6384 goto done;
6385 case ZEND_FRAMELESS_ICALL_2:
6386 op1_info = OP1_INFO();
6387 op2_info = OP2_INFO();
6388 jit_frameless_icall2(jit, opline, op1_info, op2_info);
6389 goto done;
6390 case ZEND_FRAMELESS_ICALL_3:
6391 op1_info = OP1_INFO();
6392 op2_info = OP2_INFO();
6393 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
6394 goto done;
6395 default:
6396 break;
6397 }
6398 }
6399
6400 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6401 gen_handler = 1;
6402 op1_info = OP1_INFO();
6403 op2_info = OP2_INFO();
6404 if (op1_info & MAY_BE_GUARD) {
6405 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6406 }
6407 if (op2_info & MAY_BE_GUARD) {
6408 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6409 }
6410 if (!zend_jit_trace_handler(&ctx, op_array, opline,
6411 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6412 goto jit_failure;
6413 }
6414 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
6415 if (opline->opcode == ZEND_NEW && opline->result_type != IS_UNUSED) {
6416 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6417 }
6418 if (zend_jit_may_be_polymorphic_call(opline) ||
6419 zend_jit_may_be_modified((p+1)->func, op_array)) {
6420 if (!zend_jit_init_fcall_guard(&ctx, 0, (p+1)->func, opline+1)) {
6421 goto jit_failure;
6422 }
6423 }
6424 }
6425 }
6426
6427 done:
6428 polymorphic_side_trace = 0;
6429 switch (opline->opcode) {
6430 case ZEND_DO_FCALL:
6431 case ZEND_DO_ICALL:
6432 case ZEND_DO_UCALL:
6433 case ZEND_DO_FCALL_BY_NAME:
6434 case ZEND_CALLABLE_CONVERT:
6435 frame->call_level--;
6436 }
6437
6438 if (ra) {
6439 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
6440 }
6441
6442 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6443 && STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS)) {
6444 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6445 }
6446
6447 if (opline->opcode == ZEND_ROPE_INIT) {
6448 /* clear stack slots used by rope */
6449 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6450 uint32_t count =
6451 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6452
6453 do {
6454 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6455 var++;
6456 count--;
6457 } while (count);
6458 }
6459
6460 if (ssa_op) {
6461 zend_ssa_range tmp;
6462
6463 /* Keep information about known types on abstract stack */
6464 if (ssa_op->result_def >= 0) {
6465 uint8_t type = IS_UNKNOWN;
6466
6467 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6468 || send_result) {
6469 /* we didn't set result variable */
6470 type = IS_UNKNOWN;
6471 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6472 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6473 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6474 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6475 if (opline->op1_type != IS_CONST) {
6476 /* copy */
6477 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6478 }
6479 } else if (opline->opcode == ZEND_ASSIGN) {
6480 if (opline->op2_type != IS_CONST
6481 && ssa_op->op1_use >= 0
6482 /* assignment to typed reference may cause conversion */
6483 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6484 /* copy */
6485 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6486 }
6487 } else if (opline->opcode == ZEND_POST_INC
6488 || opline->opcode == ZEND_POST_DEC) {
6489 /* copy */
6490 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6491 }
6492 if (opline->opcode == ZEND_JMP_SET
6493 || opline->opcode == ZEND_COALESCE
6494 || opline->opcode == ZEND_JMP_NULL) {
6495 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6496 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6497 } else if ((p+1)->opline != (opline + 1)) {
6498 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6499 }
6500 } else {
6501 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6502 (gen_handler || type == IS_UNKNOWN || !ra || !RA_HAS_REG(ssa_op->result_def)));
6503
6504 if (op_array->last_live_range
6505 && opline->result.var > op_array->last_var
6506 && STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) != type) {
6507 if (!gen_handler && type != IS_UNKNOWN && ra && RA_HAS_REG(ssa_op->result_def)) {
6508 uint32_t var_num = opline->result.var;
6509 uint32_t op_num = opline - op_array->opcodes;
6510 const zend_live_range *range = op_array->live_range;
6511 int j;
6512
6513 op_num += zend_jit_trace_op_len(opline);
6514 for (j = 0; j < op_array->last_live_range; range++, j++) {
6515 if (range->start > op_num) {
6516 /* further blocks will not be relevant... */
6517 break;
6518 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
6519 /* check if opcodes in range may throw */
6520 bool store_type = 0;
6521 const zend_ssa_op *next_ssa_op = ssa_op + zend_jit_trace_op_len(opline);
6522 const zend_jit_trace_rec *q = p + 1;
6523
6524 while (1) {
6525 if (q->op != ZEND_JIT_TRACE_VM) {
6526 store_type = 1;
6527 break;
6528 }
6529 op_num = q->opline - op_array->opcodes;
6530 if (op_num >= range->end || op_num < range->start) {
6531 break;
6532 }
6533 if (zend_may_throw(q->opline, next_ssa_op, op_array, ssa)) {
6534 store_type = 1;
6535 break;
6536 }
6537 next_ssa_op += zend_jit_trace_op_len(q->opline);
6538 q++;
6539 }
6540 if (store_type) {
6541 var_num = EX_VAR_TO_NUM(var_num);
6542
6543 if (!zend_jit_store_type(&ctx, var_num, type)) {
6544 return 0;
6545 }
6546 SET_STACK_TYPE(stack, var_num, type, 1);
6547 }
6548 break;
6549 }
6550 }
6551 }
6552 }
6553 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6554 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6555 }
6556 if (type != IS_UNKNOWN) {
6557 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6558 if (opline->opcode == ZEND_FETCH_THIS
6559 && delayed_fetch_this) {
6560 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_THIS);
6561 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6562 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_ZVAL_ADDREF);
6563 } else if (ra && RA_HAS_REG(ssa_op->result_def)) {
6564 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def].ref,
6565 RA_REG_FLAGS(ssa_op->result_def) & ZREG_STORE);
6566 }
6567 }
6568 }
6569
6570 if (type == IS_LONG
6571 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) {
6572 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6573 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6574 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6575 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6576 ssa->var_info[ssa_op->result_def].has_range = 1;
6577 }
6578 }
6579 if (ssa_op->op1_def >= 0
6580 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6581 || opline->result_type != IS_CV
6582 || opline->result.var != opline->op1.var)) {
6583 uint8_t type = IS_UNKNOWN;
6584
6585 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6586 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6587 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6588 } else if (opline->opcode == ZEND_ASSIGN) {
6589 if (!(OP1_INFO() & MAY_BE_REF)
6590 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6591 if (opline->op2_type != IS_CONST) {
6592 /* copy */
6593 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6594 }
6595 }
6596 } else if (opline->opcode == ZEND_SEND_VAR
6597 || opline->opcode == ZEND_CAST
6598 || opline->opcode == ZEND_QM_ASSIGN
6599 || opline->opcode == ZEND_JMP_SET
6600 || opline->opcode == ZEND_COALESCE
6601 || opline->opcode == ZEND_JMP_NULL
6602 || opline->opcode == ZEND_FE_RESET_R) {
6603 /* keep old value */
6604 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6605 }
6606 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6607 (gen_handler || type == IS_UNKNOWN || !ra ||
6608 (!RA_HAS_REG(ssa_op->op1_def) &&
6609 !(ssa->vars[ssa_op->op1_def].no_val &&
6610 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6611 (opline->opcode == ZEND_QM_ASSIGN ||
6612 opline->opcode == ZEND_SEND_VAR ||
6613 opline->opcode == ZEND_SEND_VAR_EX ||
6614 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6615 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6616 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6617 if (type != IS_UNKNOWN) {
6618 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6619 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6620 uint8_t flags = RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE;
6621
6622 if (ssa_op->op1_use >= 0) {
6623 if (opline->opcode == ZEND_SEND_VAR
6624 || opline->opcode == ZEND_CAST
6625 || opline->opcode == ZEND_QM_ASSIGN
6626 || opline->opcode == ZEND_JMP_SET
6627 || opline->opcode == ZEND_COALESCE
6628 || opline->opcode == ZEND_JMP_NULL
6629 || opline->opcode == ZEND_FE_RESET_R) {
6630 if (!RA_HAS_REG(ssa_op->op1_use)) {
6631 flags |= ZREG_LOAD;
6632 }
6633 }
6634 }
6635 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref, flags);
6636 }
6637 }
6638 if (type == IS_LONG
6639 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6640 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6641 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6642 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6643 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6644 ssa->var_info[ssa_op->op1_def].has_range = 1;
6645 }
6646 }
6647 if (ssa_op->op2_def >= 0
6648 && (opline->opcode != ZEND_ASSIGN
6649 || opline->op1_type != IS_CV
6650 || opline->op1.var != opline->op2.var)) {
6651 uint8_t type = IS_UNKNOWN;
6652
6653 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6654 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6655 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6656 } else if (opline->opcode == ZEND_ASSIGN) {
6657 /* keep old value */
6658 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6659 }
6660 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6661 (gen_handler || type == IS_UNKNOWN || !ra ||
6662 (!RA_HAS_REG(ssa_op->op2_def) &&
6663 !(ssa->vars[ssa_op->op2_def].no_val &&
6664 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6665 opline->opcode == ZEND_ASSIGN))));
6666 if (type != IS_UNKNOWN) {
6667 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6668 if (ra && RA_HAS_REG(ssa_op->op2_def)) {
6669 uint8_t flags = RA_REG_FLAGS(ssa_op->op2_def) & ZREG_STORE;
6670
6671 if (ssa_op->op2_use >= 0) {
6672 if (opline->opcode == ZEND_ASSIGN) {
6673 if (!RA_HAS_REG(ssa_op->op2_use)
6674 ) {
6675 flags |= ZREG_LOAD;
6676 }
6677 }
6678 }
6679 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def].ref, flags);
6680 }
6681 }
6682 if (type == IS_LONG
6683 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) {
6684 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6685 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6686 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6687 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6688 ssa->var_info[ssa_op->op2_def].has_range = 1;
6689 }
6690 }
6691
6692 switch (opline->opcode) {
6693 case ZEND_ASSIGN_DIM:
6694 case ZEND_ASSIGN_OBJ:
6695 case ZEND_ASSIGN_STATIC_PROP:
6696 case ZEND_ASSIGN_DIM_OP:
6697 case ZEND_ASSIGN_OBJ_OP:
6698 case ZEND_ASSIGN_STATIC_PROP_OP:
6699 case ZEND_ASSIGN_OBJ_REF:
6700 case ZEND_ASSIGN_STATIC_PROP_REF:
6701 /* OP_DATA */
6702 ssa_op++;
6703 opline++;
6704 if (ssa_op->op1_def >= 0) {
6705 uint8_t type = IS_UNKNOWN;
6706
6707 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6708 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6709 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6710 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6711 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6712 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6713 /* keep old value */
6714 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6715 }
6716 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6717 (gen_handler || type == IS_UNKNOWN || !ra || !RA_HAS_REG(ssa_op->op1_def)));
6718 if (type != IS_UNKNOWN) {
6719 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6720 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6721 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref,
6722 RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE);
6723 }
6724 }
6725 if (type == IS_LONG
6726 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6727 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6728 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6729 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6730 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6731 ssa->var_info[ssa_op->op1_def].has_range = 1;
6732 }
6733 }
6734 ssa_op++;
6735 break;
6736 case ZEND_RECV_INIT:
6737 ssa_op++;
6738 opline++;
6739 while (opline->opcode == ZEND_RECV_INIT) {
6740 if (ssa_op->result_def >= 0) {
6741 uint8_t type = IS_UNKNOWN;
6742
6743 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6744 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6745 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6746 }
6747 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6748 (gen_handler || !ra || !RA_HAS_REG(ssa_op->result_def)));
6749 if (ra && RA_HAS_REG(ssa_op->result_def)) {
6750 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def].ref,
6751 RA_REG_FLAGS(ssa_op->result_def) & ZREG_STORE);
6752 }
6753 }
6754 ssa_op++;
6755 opline++;
6756 }
6757 break;
6758 case ZEND_BIND_GLOBAL:
6759 ssa_op++;
6760 opline++;
6761 while (opline->opcode == ZEND_BIND_GLOBAL) {
6762 if (ssa_op->op1_def >= 0) {
6763 uint8_t type = IS_UNKNOWN;
6764
6765 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6766 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6767 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6768 }
6769 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6770 (gen_handler || !ra || !RA_HAS_REG(ssa_op->op1_def)));
6771 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6772 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref,
6773 RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE);
6774 }
6775 }
6776 ssa_op++;
6777 opline++;
6778 }
6779 break;
6780 default:
6781 ssa_op += zend_jit_trace_op_len(opline);
6782 break;
6783 }
6784
6785 if (send_result) {
6786 ssa_op++;
6787 p++;
6788 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6789 p++;
6790 }
6791 send_result = 0;
6792 }
6793 }
6794 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6795 call = frame->call;
6796 assert(call && &call->func->op_array == p->op_array);
6797
6798 if (opline->opcode == ZEND_DO_UCALL
6799 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6800 || opline->opcode == ZEND_DO_FCALL) {
6801
6802 frame->call_opline = opline;
6803
6804 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6805 if (opline > op_array->opcodes) {
6806 const zend_op *prev_opline = opline - 1;
6807
6808 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6809 prev_opline--;
6810 }
6811 JIT_G(current_frame) = call;
6812 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6813 || prev_opline->opcode == ZEND_SEND_UNPACK
6814 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6815 && p->op_array->num_args
6816 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6817 && ((p+1)->op == ZEND_JIT_TRACE_VM
6818 || (p+1)->op == ZEND_JIT_TRACE_END)
6819 && (TRACE_FRAME_NUM_ARGS(call) < 0
6820 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6821 && !zend_jit_trace_opline_guard(&ctx, (p+1)->opline)) {
6822 goto jit_failure;
6823 }
6824 JIT_G(current_frame) = frame;
6825 }
6826 }
6827
6828 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6829 p++;
6830 break;
6831 }
6832 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6833 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6834 TRACE_FRAME_SET_THIS_CHECKED(call);
6835 }
6836 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6837 TRACE_FRAME_SET_THIS_CHECKED(call);
6838 }
6839 op_array = (zend_op_array*)p->op_array;
6840 ctx.current_op_array = op_array;
6841 jit_extension =
6842 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6843 op_array_ssa = &jit_extension->func_info.ssa;
6844 frame->call = call->prev;
6845 call->prev = frame;
6846 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6847 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6848 } else {
6849 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6850 }
6851 JIT_G(current_frame) = frame = call;
6852 stack = frame->stack;
6853 if (ra) {
6854 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6855
6856 for (i = 0; i < op_array->last_var; i++, j++) {
6857 if (RA_HAS_REG(j) && (RA_REG_FLAGS(j) & ZREG_LOAD) != 0) {
6858 if ((ssa->var_info[j].type & MAY_BE_GUARD) != 0) {
6859 uint8_t op_type;
6860
6861 ssa->var_info[j].type &= ~MAY_BE_GUARD;
6862 op_type = concrete_type(ssa->var_info[j].type);
6863 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), op_type)) {
6864 goto jit_failure;
6865 }
6866 SET_STACK_TYPE(stack, i, op_type, 1);
6867 }
6868 if (!zend_jit_load_var(&ctx, ssa->var_info[j].type, i, j)) {
6869 goto jit_failure;
6870 }
6871 SET_STACK_REF_EX(stack, i, ra[j].ref, ZREG_LOAD);
6872 }
6873 }
6874 }
6875 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6876 op_array = (zend_op_array*)p->op_array;
6877 ctx.current_op_array = op_array;
6878 jit_extension =
6879 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6880 op_array_ssa = &jit_extension->func_info.ssa;
6881 top = frame;
6882 if (frame->prev) {
6883 checked_stack = frame->old_checked_stack;
6884 peek_checked_stack = frame->old_peek_checked_stack;
6885 frame = frame->prev;
6886 stack = frame->stack;
6887 ZEND_ASSERT(&frame->func->op_array == op_array);
6888 } else {
6889 frame = zend_jit_trace_ret_frame(frame, op_array);
6890 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6891 frame->used_stack = checked_stack = peek_checked_stack = 0;
6892 stack = frame->stack;
6893 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6894 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6895
6896 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6897 /* Initialize abstract stack using SSA */
6898 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6899 && has_concrete_type(ssa->var_info[j].type)) {
6900 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6901 } else {
6902 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6903 }
6904 }
6905 if (ra) {
6906 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6907 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6908 if (RA_HAS_REG(j) && (RA_REG_FLAGS(j) & ZREG_LOAD) != 0) {
6909 if (!zend_jit_load_var(&ctx, ssa->var_info[j].type, i, j)) {
6910 goto jit_failure;
6911 }
6912 SET_STACK_REF_EX(stack, i, ra[j].ref, ZREG_LOAD);
6913 }
6914 }
6915 }
6916 } else {
6917 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6918 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6919 }
6920 }
6921 opline = NULL;
6922 }
6923 JIT_G(current_frame) = frame;
6924 if (res_type != IS_UNKNOWN
6925 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6926 const zend_op *opline = (p+1)->opline - 1;
6927 if (opline->result_type != IS_UNUSED) {
6928 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6929 }
6930 }
6931 res_type = IS_UNKNOWN;
6932 } else if (p->op == ZEND_JIT_TRACE_END) {
6933 break;
6934 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6935 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6936 int num_args = -1;
6937
6938 if (init_opline
6939 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6940 num_args = init_opline->extended_value;
6941 }
6942
6943 call = top;
6944 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6945 call->prev = frame->call;
6946 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6947 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6948 if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
6949 TRACE_FRAME_SET_CLOSURE_CALL(call);
6950 }
6951 }
6952 if (init_opline) {
6953 if (init_opline->opcode != ZEND_NEW
6954 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6955 || init_opline->op1_type == IS_UNDEF
6956 || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6957 && ssa_op
6958 && (ssa_op-1)->op1_use >=0
6959 && ssa->var_info[(ssa_op-1)->op1_use].delayed_fetch_this))
6960 && (init_opline->opcode != ZEND_INIT_USER_CALL
6961 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6962 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6963 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6964 ) {
6965 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6966 } else if (init_opline->opcode == ZEND_NEW
6967 || (init_opline->opcode == ZEND_INIT_METHOD_CALL
6968 && init_opline->op1_type != IS_UNDEF
6969 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6970 && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) {
6971 TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call);
6972 }
6973 }
6974 frame->call = call;
6975 top = zend_jit_trace_call_frame(top, p->op_array);
6976 if (p->func) {
6977 if (p->func->type == ZEND_USER_FUNCTION) {
6978 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6979 zend_jit_op_array_trace_extension *jit_extension =
6980 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6981
6982 i = 0;
6983 while (i < p->op_array->num_args) {
6984 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6985 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6986 i++;
6987 }
6988 while (i < p->op_array->last_var) {
6989 if (jit_extension
6990 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6991 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6992 } else {
6993 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6994 }
6995 i++;
6996 }
6997 while (i < p->op_array->last_var + p->op_array->T) {
6998 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6999 i++;
7000 }
7001 } else {
7002 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
7003 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7004 }
7005 }
7006 } else {
7007 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
7008 for (i = 0; i < p->op_array->num_args; i++) {
7009 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7010 }
7011 }
7012 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
7013 int skip_guard = 0;
7014
7015 if (init_opline) {
7016 zend_call_info *call_info = jit_extension->func_info.callee_info;
7017
7018 while (call_info) {
7019 if (call_info->caller_init_opline == init_opline
7020 && !call_info->is_prototype) {
7021 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
7022 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
7023 && init_opline->op1_type != IS_CONST) {
7024 break;
7025 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
7026 break;
7027 }
7028 }
7029 skip_guard = 1;
7030 break;
7031 }
7032 call_info = call_info->next_callee;
7033 }
7034 if (!skip_guard
7035 && !zend_jit_may_be_polymorphic_call(init_opline)
7036 && !zend_jit_may_be_modified(p->func, op_array)) {
7037 skip_guard = 1;
7038 }
7039 }
7040
7041 if (!skip_guard) {
7042 if (!opline) {
7043 zend_jit_trace_rec *q = p + 1;
7044 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
7045 q++;
7046 }
7047 opline = q->opline;
7048 ZEND_ASSERT(opline != NULL);
7049 }
7050 if (!zend_jit_init_fcall_guard(&ctx,
7051 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
7052 goto jit_failure;
7053 }
7054 }
7055 }
7056 }
7057 call->old_checked_stack = checked_stack;
7058 call->old_peek_checked_stack = peek_checked_stack;
7059 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
7060 frame->call_level++;
7061 call->used_stack = checked_stack = peek_checked_stack = 0;
7062 } else {
7063 if (p->func) {
7064 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
7065 } else {
7066 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
7067 }
7068 switch (init_opline->opcode) {
7069 case ZEND_INIT_FCALL:
7070 case ZEND_INIT_FCALL_BY_NAME:
7071 case ZEND_INIT_NS_FCALL_BY_NAME:
7072 case ZEND_INIT_METHOD_CALL:
7073 case ZEND_INIT_DYNAMIC_CALL:
7074 //case ZEND_INIT_STATIC_METHOD_CALL:
7075 //case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
7076 //case ZEND_INIT_USER_CALL:
7077 //case ZEND_NEW:
7078 checked_stack += call->used_stack;
7079 if (checked_stack > peek_checked_stack) {
7080 peek_checked_stack = checked_stack;
7081 }
7082 break;
7083 default:
7084 checked_stack = peek_checked_stack = 0;
7085 }
7086 }
7087 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7088 call = frame->call;
7089 if (call) {
7090 checked_stack = call->old_checked_stack;
7091 peek_checked_stack = call->old_peek_checked_stack;
7092 top = call;
7093 frame->call = call->prev;
7094 }
7095 } else {
7096 ZEND_UNREACHABLE();
7097 }
7098 }
7099
7100 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
7101
7102 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7103
7104 if (!parent_trace && zend_jit_trace_uses_initial_ip(&ctx)) {
7105 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
7106 }
7107
7108 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
7109 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7110 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7111 if (ra) {
7112 zend_ssa_phi *phi = ssa->blocks[1].phis;
7113
7114 while (phi) {
7115 if (RA_HAS_REG(phi->sources[1])
7116 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
7117 && (RA_REG_FLAGS(phi->sources[1]) & (ZREG_LOAD|ZREG_STORE)) == 0) {
7118
7119 if (!RA_HAS_REG(phi->ssa_var)
7120 || (RA_REG_FLAGS(phi->ssa_var) & (ZREG_LOAD|ZREG_STORE)) == 0) {
7121 /* Store actual type to memory to avoid deoptimization mistakes */
7122 zend_jit_store_var_type(&ctx, phi->var, STACK_TYPE(stack, phi->var));
7123 }
7124 }
7125 phi = phi->next;
7126 }
7127 }
7128 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7129 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7130 && !zend_jit_set_ip(&ctx, p->opline)) {
7131 goto jit_failure;
7132 }
7133 }
7134 t->link = ZEND_JIT_TRACE_NUM;
7135 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7136 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
7137 }
7138 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
7139 const void *timeout_exit_addr = NULL;
7140
7141 t->flags |= ZEND_JIT_TRACE_LOOP;
7142
7143 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7144 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7145 || (ra
7146 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
7147 /* Deoptimize to the first instruction of the loop */
7148 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
7149
7150 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7151 if (!timeout_exit_addr) {
7152 goto jit_failure;
7153 }
7154 } else {
7155 timeout_exit_addr = zend_jit_stub_handlers[jit_stub_interrupt_handler];
7156 }
7157 }
7158
7159 zend_jit_trace_end_loop(&ctx, jit->trace_loop_ref, timeout_exit_addr); /* jump back to start of the trace loop */
7160 }
7161 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7162 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7163 if (ra
7164 && (p-1)->op != ZEND_JIT_TRACE_ENTER
7165 && (p-1)->op != ZEND_JIT_TRACE_BACK
7166 && opline->opcode != ZEND_DO_UCALL
7167 && opline->opcode != ZEND_DO_FCALL
7168 && opline->opcode != ZEND_DO_FCALL_BY_NAME
7169 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
7170 for (i = 0; i < op_array->last_var + op_array->T; i++) {
7171 int32_t ref = STACK_REF(stack, i);
7172 uint8_t type = STACK_TYPE(stack, i);
7173
7174 if (ref && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
7175 if (!zend_jit_store_ref(jit, 1 << type, i, ref, STACK_MEM_TYPE(stack, i) != type)) {
7176 goto jit_failure;
7177 }
7178 SET_STACK_TYPE(stack, i, type, 1);
7179 } else if (i < op_array->last_var
7180 && type != IS_UNKNOWN
7181 && type != STACK_MEM_TYPE(stack, i)
7182 && zend_jit_trace_must_store_type(op_array, op_array_ssa, opline - op_array->opcodes, i, type)) {
7183 if (!zend_jit_store_type(jit, i, type)) {
7184 return 0;
7185 }
7186 SET_STACK_TYPE(stack, i, type, 1);
7187 }
7188 CLEAR_STACK_REF(stack, i);
7189 }
7190 }
7191 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
7192 const void *timeout_exit_addr = NULL;
7193
7194 t->link = zend_jit_find_trace(p->opline->handler);
7195 if (t->link == 0) {
7196 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
7197 * by zend_jit_trace_exit() in another thread after this
7198 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
7199 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
7200 * the "_counter_handler" functions, and these are not registered
7201 * tracer functions */
7202 goto jit_failure;
7203 }
7204 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7205 && !zend_jit_set_ip(&ctx, p->opline)) {
7206 goto jit_failure;
7207 }
7208 if (!parent_trace && zend_jit_trace_uses_initial_ip(&ctx)) {
7209 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
7210 }
7211 if (parent_trace
7212 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
7213 && zend_jit_traces[parent_trace].root == t->link) {
7214 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
7215 uint32_t exit_point;
7216
7217 for (i = 0; i < op_array->last_var + op_array->T; i++) {
7218 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
7219 }
7220 exit_point = zend_jit_trace_get_exit_point(zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM);
7221 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7222 if (!timeout_exit_addr) {
7223 goto jit_failure;
7224 }
7225 } else {
7226 timeout_exit_addr = zend_jit_stub_handlers[jit_stub_interrupt_handler];
7227 }
7228 }
7229 zend_jit_trace_link_to_root(&ctx, &zend_jit_traces[t->link], timeout_exit_addr);
7230 } else {
7231 zend_jit_trace_return(&ctx, 0, NULL);
7232 }
7233 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
7234 zend_jit_trace_return(&ctx, 0, NULL);
7235 } else {
7236 // TODO: not implemented ???
7237 ZEND_ASSERT(0 && p->stop);
7238 }
7239
7240 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7241 goto jit_failure;
7242 }
7243
7244 handler = zend_jit_finish(&ctx);
7245
7246 if (handler) {
7247 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
7248 const zend_op_array *rec_op_array;
7249
7250 rec_op_array = op_array = trace_buffer->op_array;
7251 jit_extension =
7252 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7253 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7254 for (;;p++) {
7255 if (p->op == ZEND_JIT_TRACE_VM) {
7256 opline = p->opline;
7257 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7258 if (p->op_array == rec_op_array) {
7259 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7260 }
7261 op_array = p->op_array;
7262 jit_extension =
7263 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7264 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7265 op_array = p->op_array;
7266 jit_extension =
7267 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7268 } else if (p->op == ZEND_JIT_TRACE_END) {
7269 break;
7270 }
7271 }
7272 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7273 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7274 if (opline
7275 && (opline->opcode == ZEND_DO_UCALL
7276 || opline->opcode == ZEND_DO_FCALL
7277 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7278 || opline->opcode == ZEND_YIELD
7279 || opline->opcode == ZEND_YIELD_FROM
7280 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7281 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7282 }
7283 if (JIT_G(current_frame)
7284 && JIT_G(current_frame)->prev) {
7285 frame = JIT_G(current_frame)->prev;
7286 do {
7287 if (frame->call_opline) {
7288 op_array = &frame->func->op_array;
7289 jit_extension =
7290 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7291 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7292 }
7293 frame = frame->prev;
7294 } while (frame);
7295 }
7296 }
7297 }
7298
7299 jit_failure:
7300 zend_jit_free_ctx(&ctx);
7301
7302 if (name) {
7303 zend_string_release(name);
7304 }
7305
7306 jit_cleanup:
7307 /* Clean up used op_arrays */
7308 while (num_op_arrays > 0) {
7309 op_array = op_arrays[--num_op_arrays];
7310 jit_extension =
7311 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7312
7313 jit_extension->func_info.num = 0;
7314 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7315 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7316 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7317 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7318 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7319 }
7320
7321 zend_arena_release(&CG(arena), checkpoint);
7322
7323 JIT_G(current_frame) = NULL;
7324 JIT_G(current_trace) = NULL;
7325
7326 return handler;
7327 }
7328
zend_jit_trace_escape_name(uint32_t trace_num,uint32_t exit_num)7329 static zend_string *zend_jit_trace_escape_name(uint32_t trace_num, uint32_t exit_num)
7330 {
7331 smart_str buf = {0};
7332
7333 smart_str_appends(&buf," ESCAPE-");
7334 smart_str_append_long(&buf, (zend_long)trace_num);
7335 smart_str_appendc(&buf, '-');
7336 smart_str_append_long(&buf, (zend_long)exit_num);
7337 smart_str_0(&buf);
7338 return buf.s;
7339 }
7340
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7341 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7342 {
7343 const void *handler = NULL;
7344 zend_jit_ctx ctx;
7345 zend_string *name;
7346 void *checkpoint;
7347 const zend_op *opline;
7348 uint32_t stack_size;
7349 zend_jit_trace_stack *stack;
7350 bool original_handler = 0;
7351
7352 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7353 return zend_jit_stub_handlers[jit_stub_trace_escape];
7354 }
7355
7356 name = zend_jit_trace_escape_name(trace_num, exit_num);
7357
7358 if (!zend_jit_deoptimizer_start(&ctx, name, trace_num, exit_num)) {
7359 zend_string_release(name);
7360 return NULL;
7361 }
7362
7363 checkpoint = zend_arena_checkpoint(CG(arena));;
7364
7365 /* Deoptimization */
7366 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7367 stack = zend_jit_traces[trace_num].exit_info[exit_num].stack_size ?
7368 zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset :
7369 NULL;
7370
7371 if (!zend_jit_trace_deoptimization(&ctx,
7372 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7373 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7374 stack, stack_size, NULL, NULL,
7375 zend_jit_traces[trace_num].constants,
7376 zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg,
7377 0)) {
7378 goto jit_failure;
7379 }
7380
7381 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7382 if (opline) {
7383 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7384 zend_jit_op_array_trace_extension *jit_extension =
7385 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(zend_jit_traces[zend_jit_traces[trace_num].root].op_array);
7386
7387 if (ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->orig_handler != opline->handler) {
7388 /* prevent endless loop */
7389 original_handler = 1;
7390 }
7391 }
7392 zend_jit_set_ip_ex(&ctx, opline, original_handler);
7393 }
7394
7395 zend_jit_trace_return(&ctx, original_handler, opline);
7396
7397 handler = zend_jit_finish(&ctx);
7398
7399 jit_failure:
7400 zend_jit_free_ctx(&ctx);
7401 zend_string_release(name);
7402 zend_arena_release(&CG(arena), checkpoint);
7403 return handler;
7404 }
7405
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7406 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7407 {
7408 zend_jit_trace_stop ret;
7409 const void *handler;
7410 uint8_t orig_trigger;
7411 zend_jit_trace_info *t = NULL;
7412 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7413 bool do_bailout = 0;
7414
7415 zend_shared_alloc_lock();
7416
7417 /* Checks under lock */
7418 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7419 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7420 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7421 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7422 } else {
7423 zend_try {
7424 SHM_UNPROTECT();
7425 zend_jit_unprotect();
7426
7427 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7428
7429 t->id = ZEND_JIT_TRACE_NUM;
7430 t->root = ZEND_JIT_TRACE_NUM;
7431 t->parent = 0;
7432 t->link = 0;
7433 t->exit_count = 0;
7434 t->child_count = 0;
7435 t->stack_map_size = 0;
7436 t->flags = 0;
7437 t->polymorphism = 0;
7438 t->jmp_table_size = 0;
7439 t->op_array = trace_buffer[0].op_array;
7440 t->opline = trace_buffer[1].opline;
7441 t->exit_info = exit_info;
7442 t->stack_map = NULL;
7443 t->consts_count = 0;
7444 t->constants = NULL;
7445
7446 orig_trigger = JIT_G(trigger);
7447 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7448
7449 handler = zend_jit_trace(trace_buffer, 0, 0);
7450
7451 JIT_G(trigger) = orig_trigger;
7452
7453 if (handler) {
7454 zend_jit_trace_exit_info *shared_exit_info = NULL;
7455
7456 t->exit_info = NULL;
7457 if (t->exit_count) {
7458 /* reallocate exit_info into shared memory */
7459 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7460 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7461
7462 if (!shared_exit_info) {
7463 if (t->stack_map) {
7464 efree(t->stack_map);
7465 t->stack_map = NULL;
7466 }
7467 if (t->constants) {
7468 efree(t->constants);
7469 t->constants = NULL;
7470 }
7471 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7472 goto exit;
7473 }
7474 memcpy(shared_exit_info, exit_info,
7475 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7476 t->exit_info = shared_exit_info;
7477 }
7478
7479 if (t->stack_map_size) {
7480 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7481 if (!shared_stack_map) {
7482 efree(t->stack_map);
7483 t->stack_map = NULL;
7484 if (t->constants) {
7485 efree(t->constants);
7486 t->constants = NULL;
7487 }
7488 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7489 goto exit;
7490 }
7491 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7492 efree(t->stack_map);
7493 t->stack_map = shared_stack_map;
7494 }
7495
7496 if (t->consts_count) {
7497 zend_jit_exit_const *constants = (zend_jit_exit_const*)zend_shared_alloc(t->consts_count * sizeof(zend_jit_exit_const));
7498 if (!constants) {
7499 efree(t->constants);
7500 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7501 goto exit;
7502 }
7503 memcpy(constants, t->constants, t->consts_count * sizeof(zend_jit_exit_const));
7504 efree(t->constants);
7505 t->constants = constants;
7506 }
7507
7508 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7509 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7510
7511 ((zend_op*)opline)->handler = handler;
7512
7513 ZEND_JIT_TRACE_NUM++;
7514 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7515
7516 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7517 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7518 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7519 if (t->stack_map) {
7520 efree(t->stack_map);
7521 t->stack_map = NULL;
7522 }
7523 if (t->constants) {
7524 efree(t->constants);
7525 t->constants = NULL;
7526 }
7527 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7528 } else {
7529 if (t->stack_map) {
7530 efree(t->stack_map);
7531 t->stack_map = NULL;
7532 }
7533 if (t->constants) {
7534 efree(t->constants);
7535 t->constants = NULL;
7536 }
7537 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7538 }
7539
7540 exit:;
7541 } zend_catch {
7542 do_bailout = 1;
7543 } zend_end_try();
7544
7545 zend_jit_protect();
7546 SHM_PROTECT();
7547 }
7548
7549 zend_shared_alloc_unlock();
7550
7551 if (do_bailout) {
7552 zend_bailout();
7553 }
7554
7555 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7556 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7557 && t->exit_count > 0) {
7558 zend_jit_dump_exit_info(t);
7559 }
7560
7561 return ret;
7562 }
7563
7564 /* Set counting handler back to original VM handler. */
zend_jit_stop_hot_trace_counters(zend_op_array * op_array)7565 static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array)
7566 {
7567 zend_jit_op_array_trace_extension *jit_extension;
7568 uint32_t i;
7569
7570 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7571 for (i = 0; i < op_array->last; i++) {
7572 /* Opline with Jit-ed code handler is skipped. */
7573 if (jit_extension->trace_info[i].trace_flags &
7574 (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7575 continue;
7576 }
7577 if (jit_extension->trace_info[i].trace_flags &
7578 (ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_START_RETURN)) {
7579 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7580 }
7581 }
7582 }
7583
7584 /* Get the tracing op_array. */
zend_jit_stop_persistent_op_array(zend_op_array * op_array)7585 static void zend_jit_stop_persistent_op_array(zend_op_array *op_array)
7586 {
7587 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
7588 if (!func_info) {
7589 return;
7590 }
7591 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
7592 zend_jit_stop_hot_trace_counters(op_array);
7593 }
7594 }
7595
7596 /* Get all op_arrays with counter handler. */
zend_jit_stop_persistent_script(zend_persistent_script * script)7597 static void zend_jit_stop_persistent_script(zend_persistent_script *script)
7598 {
7599 zend_class_entry *ce;
7600 zend_op_array *op_array;
7601
7602 zend_jit_stop_persistent_op_array(&script->script.main_op_array);
7603
7604 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
7605 zend_jit_stop_persistent_op_array(op_array);
7606 } ZEND_HASH_FOREACH_END();
7607
7608 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
7609 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
7610 if (op_array->type == ZEND_USER_FUNCTION) {
7611 zend_jit_stop_persistent_op_array(op_array);
7612 }
7613 } ZEND_HASH_FOREACH_END();
7614 } ZEND_HASH_FOREACH_END();
7615 }
7616
7617 /* Get all scripts which are accelerated by JIT */
zend_jit_stop_counter_handlers(void)7618 static void zend_jit_stop_counter_handlers(void)
7619 {
7620 if (ZCSG(jit_counters_stopped)) {
7621 return;
7622 }
7623
7624 zend_shared_alloc_lock();
7625 /* mprotect has an extreme overhead, avoid calls to it for every function. */
7626 SHM_UNPROTECT();
7627 if (!ZCSG(jit_counters_stopped)) {
7628 ZCSG(jit_counters_stopped) = true;
7629 for (uint32_t i = 0; i < ZCSG(hash).max_num_entries; i++) {
7630 zend_accel_hash_entry *cache_entry;
7631 for (cache_entry = ZCSG(hash).hash_table[i]; cache_entry; cache_entry = cache_entry->next) {
7632 zend_persistent_script *script;
7633 if (cache_entry->indirect) continue;
7634 script = (zend_persistent_script *)cache_entry->data;
7635 zend_jit_stop_persistent_script(script);
7636 }
7637 }
7638 }
7639 SHM_PROTECT();
7640 zend_shared_alloc_unlock();
7641 }
7642
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7643 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7644 {
7645 zend_shared_alloc_lock();
7646
7647 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7648 SHM_UNPROTECT();
7649 zend_jit_unprotect();
7650
7651 ((zend_op*)opline)->handler =
7652 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7653
7654 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7655
7656 zend_jit_protect();
7657 SHM_PROTECT();
7658 }
7659
7660 zend_shared_alloc_unlock();
7661 }
7662
zend_jit_blacklist_function(zend_op_array * op_array)7663 ZEND_EXT_API void zend_jit_blacklist_function(zend_op_array *op_array) {
7664 zend_jit_op_array_trace_extension *jit_extension = (zend_jit_op_array_trace_extension *)ZEND_FUNC_INFO(op_array);
7665 if (!jit_extension || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) {
7666 return;
7667 }
7668
7669 zend_shared_alloc_lock();
7670 SHM_UNPROTECT();
7671 zend_jit_unprotect();
7672
7673 zend_jit_stop_persistent_op_array(op_array);
7674 jit_extension->func_info.flags &= ~ZEND_FUNC_JIT_ON_HOT_TRACE;
7675
7676 zend_jit_protect();
7677 SHM_PROTECT();
7678 zend_shared_alloc_unlock();
7679 }
7680
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7681 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7682 {
7683 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7684 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7685 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7686 uint32_t cache_slot = JIT_G(bad_root_slot);
7687 uint32_t i;
7688
7689 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7690 if (cache_opline[i] == opline) {
7691 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7692 cache_opline[i] = NULL;
7693 return 1;
7694 } else {
7695 #if 0
7696 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7697 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7698 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7699 }
7700 #endif
7701 cache_count[i]++;
7702 cache_stop[i] = stop;
7703 return 0;
7704 }
7705 }
7706 }
7707 i = cache_slot;
7708 cache_opline[i] = opline;
7709 cache_count[i] = 1;
7710 cache_stop[i] = stop;
7711 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7712 JIT_G(bad_root_slot) = cache_slot;
7713 return 0;
7714 }
7715
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7716 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7717 {
7718 zend_jit_trace_rec *p = trace_buffer;
7719 const zend_op_array *op_array;
7720 const zend_op *opline;
7721 uint32_t level = 1 + trace_buffer[0].level;
7722 int idx, len, i, v, vars_count, call_level;
7723
7724 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7725 op_array = p->op_array;
7726 p += ZEND_JIT_TRACE_START_REC_SIZE;
7727 idx = 0;
7728 call_level = 0;
7729
7730 if (tssa && tssa->var_info) {
7731 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7732 vars_count = op_array->last_var;
7733 } else {
7734 vars_count = op_array->last_var + op_array->T;
7735 }
7736 for (i = 0; i < vars_count; i++) {
7737 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7738 fprintf(stderr, " %*c;", level, ' ');
7739 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7740 fprintf(stderr, "\n");
7741 }
7742 }
7743 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7744 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7745 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7746 zend_ssa_phi *p = tssa->blocks[1].phis;
7747
7748 fprintf(stderr, "LOOP:\n");
7749
7750 while (p) {
7751 fprintf(stderr, " ;");
7752 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7753 fprintf(stderr, " = Phi(");
7754 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7755 fprintf(stderr, ", ");
7756 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7757 fprintf(stderr, ")\n");
7758 p = p->next;
7759 }
7760 }
7761 }
7762
7763 while (1) {
7764 if (p->op == ZEND_JIT_TRACE_VM) {
7765 uint8_t op1_type, op2_type, op3_type;
7766
7767 opline = p->opline;
7768 fprintf(stderr, "%04d%*c",
7769 (int)(opline - op_array->opcodes),
7770 level, ' ');
7771 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7772
7773 op1_type = p->op1_type;
7774 op2_type = p->op2_type;
7775 op3_type = p->op3_type;
7776 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7777 fprintf(stderr, " ;");
7778 if (op1_type != IS_UNKNOWN) {
7779 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7780 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7781 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7782 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7783 p++;
7784 fprintf(stderr, " op1(%sobject of class %s)", ref,
7785 ZSTR_VAL(p->ce->name));
7786 } else {
7787 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7788 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7789 }
7790 }
7791 if (op2_type != IS_UNKNOWN) {
7792 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7793 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7794 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7795 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7796 p++;
7797 fprintf(stderr, " op2(%sobject of class %s)", ref,
7798 ZSTR_VAL(p->ce->name));
7799 } else {
7800 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7801 fprintf(stderr, " op2(%s%s)", ref, type);
7802 }
7803 }
7804 if (op3_type != IS_UNKNOWN) {
7805 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7806 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7807 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7808 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7809 fprintf(stderr, " op3(%s%s)", ref, type);
7810 }
7811 }
7812 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7813 uint8_t val_type;
7814 const char *type;
7815
7816 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7817 fprintf(stderr, " ;");
7818 }
7819 p++;
7820 val_type = p->op1_type;
7821
7822 if (val_type == IS_UNDEF) {
7823 type = "undef";
7824 } else if (val_type == IS_REFERENCE) {
7825 type = "ref";
7826 } else {
7827 type = zend_get_type_by_const(val_type);
7828 }
7829 fprintf(stderr, " val(%s)", type);
7830 }
7831 fprintf(stderr, "\n");
7832 idx++;
7833
7834 len = zend_jit_trace_op_len(opline);
7835 while (len > 1) {
7836 opline++;
7837 fprintf(stderr, "%04d%*c;",
7838 (int)(opline - op_array->opcodes),
7839 level, ' ');
7840 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7841 idx++;
7842 len--;
7843 fprintf(stderr, "\n");
7844 }
7845 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7846 op_array = p->op_array;
7847 fprintf(stderr, " %*c>enter %s%s%s\n",
7848 level, ' ',
7849 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7850 op_array->scope ? "::" : "",
7851 op_array->function_name ?
7852 ZSTR_VAL(op_array->function_name) :
7853 ZSTR_VAL(op_array->filename));
7854 level++;
7855 if (tssa && tssa->var_info) {
7856 call_level++;
7857 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7858 vars_count = op_array->last_var;
7859 for (i = 0; i < vars_count; i++, v++) {
7860 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7861 fprintf(stderr, " %*c;", level, ' ');
7862 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7863 fprintf(stderr, "\n");
7864 }
7865 }
7866 }
7867 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7868 op_array = p->op_array;
7869 level--;
7870 fprintf(stderr, " %*c<back %s%s%s\n",
7871 level, ' ',
7872 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7873 op_array->scope ? "::" : "",
7874 op_array->function_name ?
7875 ZSTR_VAL(op_array->function_name) :
7876 ZSTR_VAL(op_array->filename));
7877 if (tssa && tssa->var_info) {
7878 if (call_level == 0) {
7879 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7880 vars_count = op_array->last_var + op_array->T;
7881 for (i = 0; i < vars_count; i++, v++) {
7882 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7883 fprintf(stderr, " %*c;", level, ' ');
7884 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7885 fprintf(stderr, "\n");
7886 }
7887 }
7888 } else {
7889 call_level--;
7890 }
7891 }
7892 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7893 if (p->func != (zend_function*)&zend_pass_function) {
7894 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7895 level, ' ',
7896 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7897 (p->func && p->func->common.scope) ? "::" : "",
7898 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7899 } else {
7900 fprintf(stderr, " %*c>skip\n",
7901 level, ' ');
7902 }
7903 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7904 if (p->func != (zend_function*)&zend_pass_function) {
7905 fprintf(stderr, " %*c>call %s%s%s\n",
7906 level, ' ',
7907 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7908 p->func->common.scope ? "::" : "",
7909 ZSTR_VAL(p->func->common.function_name));
7910 } else {
7911 fprintf(stderr, " %*c>skip\n",
7912 level, ' ');
7913 }
7914 } else if (p->op == ZEND_JIT_TRACE_END) {
7915 break;
7916 }
7917 p++;
7918 }
7919 }
7920
zend_jit_dump_exit_info(zend_jit_trace_info * t)7921 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7922 {
7923 int i, j;
7924
7925 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7926 for (i = 0; i < t->exit_count; i++) {
7927 const zend_op_array *op_array = t->exit_info[i].op_array;
7928 uint32_t stack_size = t->exit_info[i].stack_size;
7929 zend_jit_trace_stack *stack = t->exit_info[i].stack_size ? t->stack_map + t->exit_info[i].stack_offset : NULL;
7930
7931 fprintf(stderr, " exit_%d:", i);
7932 if (t->exit_info[i].opline) {
7933 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7934 } else {
7935 fprintf(stderr, " ----/");
7936 }
7937 if (t->exit_info[i].stack_size) {
7938 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7939 } else {
7940 fprintf(stderr, "----/0");
7941 }
7942 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7943 fprintf(stderr, "/VM");
7944 }
7945 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7946 fprintf(stderr, "/CALL");
7947 }
7948 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7949 fprintf(stderr, "/POLY");
7950 if (t->exit_info[i].flags & ZEND_JIT_EXIT_METHOD_CALL) {
7951 fprintf(stderr, "(%s, %s)",
7952 t->exit_info[i].poly_func_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_func_reg) : "?",
7953 t->exit_info[i].poly_this_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_this_reg) : "?");
7954 }
7955 }
7956 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7957 fprintf(stderr, "/FREE_OP1");
7958 }
7959 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7960 fprintf(stderr, "/FREE_OP2");
7961 }
7962 for (j = 0; j < stack_size; j++) {
7963 uint8_t type = STACK_TYPE(stack, j);
7964 if (type != IS_UNKNOWN) {
7965 fprintf(stderr, " ");
7966 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7967 fprintf(stderr, ":");
7968 if (type == IS_UNDEF) {
7969 fprintf(stderr, "undef");
7970 } else {
7971 fprintf(stderr, "%s", zend_get_type_by_const(type));
7972 }
7973 if (STACK_FLAGS(stack, j) == ZREG_CONST) {
7974 if (type == IS_LONG) {
7975 fprintf(stderr, "(" ZEND_LONG_FMT ")", (zend_long)t->constants[STACK_REF(stack, j)].i);
7976 } else if (type == IS_DOUBLE) {
7977 fprintf(stderr, "(%g)", t->constants[STACK_REF(stack, j)].d);
7978 } else {
7979 ZEND_UNREACHABLE();
7980 }
7981 } else if (STACK_FLAGS(stack, j) == ZREG_TYPE_ONLY) {
7982 fprintf(stderr, "(type_only)");
7983 } else if (STACK_FLAGS(stack, j) == ZREG_THIS) {
7984 fprintf(stderr, "(this)");
7985 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_ADDREF) {
7986 fprintf(stderr, "(zval_try_addref)");
7987 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
7988 fprintf(stderr, "zval_copy(%s)", zend_reg_name(STACK_REG(stack, j)));
7989 } else if (STACK_FLAGS(stack, j) & ZREG_SPILL_SLOT) {
7990 if (STACK_REG(stack, j) == ZREG_NONE) {
7991 fprintf(stderr, "(spill=0x%x", STACK_REF(stack, j));
7992 } else {
7993 fprintf(stderr, "(spill=0x%x(%s)", STACK_REF(stack, j), zend_reg_name(STACK_REG(stack, j)));
7994 }
7995 if (STACK_FLAGS(stack, j) != 0) {
7996 fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
7997 }
7998 fprintf(stderr, ")");
7999 } else if (STACK_REG(stack, j) != ZREG_NONE) {
8000 fprintf(stderr, "(%s", zend_reg_name(STACK_REG(stack, j)));
8001 if (STACK_FLAGS(stack, j) != 0) {
8002 fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
8003 }
8004 fprintf(stderr, ")");
8005 }
8006 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_ADDREF) {
8007 fprintf(stderr, ":unknown(zval_try_addref)");
8008 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
8009 fprintf(stderr, " ");
8010 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
8011 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name(STACK_REG(stack, j)));
8012 }
8013 }
8014 fprintf(stderr, "\n");
8015 }
8016 }
8017
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)8018 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
8019 {
8020 const zend_op *orig_opline;
8021 zend_jit_trace_stop stop;
8022 int ret = 0;
8023 zend_op_array *op_array;
8024 zend_jit_op_array_trace_extension *jit_extension;
8025 size_t offset;
8026 uint32_t trace_num;
8027 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8028
8029 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8030 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
8031 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8032
8033 repeat:
8034 trace_num = ZEND_JIT_TRACE_NUM;
8035 orig_opline = opline;
8036 op_array = &EX(func)->op_array;
8037 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8038 offset = jit_extension->offset;
8039
8040 EX(opline) = opline;
8041
8042 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
8043 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
8044 return 0;
8045 }
8046
8047 if (JIT_G(tracing)) {
8048 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
8049 return 0;
8050 }
8051
8052 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8053 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8054 trace_num,
8055 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8056 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8057 EX(func)->op_array.scope ? "::" : "",
8058 EX(func)->op_array.function_name ?
8059 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8060 ZSTR_VAL(EX(func)->op_array.filename),
8061 opline->lineno);
8062 }
8063
8064 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8065 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8066 zend_jit_stop_counter_handlers();
8067 goto abort;
8068 }
8069
8070 JIT_G(tracing) = 1;
8071 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
8072 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0, 0);
8073 JIT_G(tracing) = 0;
8074
8075 if (stop & ZEND_JIT_TRACE_HALT) {
8076 ret = -1;
8077 }
8078 stop &= ~ZEND_JIT_TRACE_HALT;
8079
8080 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
8081 orig_opline = trace_buffer[1].opline;
8082 op_array = (zend_op_array*)trace_buffer[0].op_array;
8083 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8084 offset = jit_extension->offset;
8085 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8086 const zend_op_array *op_array = trace_buffer[0].op_array;
8087 const zend_op *opline = trace_buffer[1].opline;
8088 zend_jit_op_array_trace_extension *jit_extension =
8089 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8090 size_t offset = jit_extension->offset;
8091
8092 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8093 trace_num,
8094 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8095 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8096 op_array->scope ? "::" : "",
8097 op_array->function_name ?
8098 ZSTR_VAL(op_array->function_name) : "$main",
8099 ZSTR_VAL(op_array->filename),
8100 opline->lineno);
8101 }
8102 }
8103
8104 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8105 zend_jit_dump_trace(trace_buffer, NULL);
8106 }
8107
8108 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8109 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8110 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8111 uint32_t idx = trace_buffer[1].last;
8112 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
8113 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8114 trace_num,
8115 link_to);
8116 } else {
8117 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8118 trace_num,
8119 zend_jit_trace_stop_description[stop]);
8120 }
8121 }
8122 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
8123 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8124 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8125 fprintf(stderr, "---- TRACE %d %s\n",
8126 trace_num,
8127 zend_jit_trace_stop_description[stop]);
8128 }
8129 } else {
8130 goto abort;
8131 }
8132 } else {
8133 abort:
8134 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8135 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8136 trace_num,
8137 zend_jit_trace_stop_description[stop]);
8138 }
8139 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8140 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
8141 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8142 fprintf(stderr, "---- TRACE %d blacklisted\n",
8143 trace_num);
8144 }
8145 zend_jit_blacklist_root_trace(orig_opline, offset);
8146 }
8147 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8148 execute_data = EG(current_execute_data);
8149 opline = EX(opline);
8150 goto repeat;
8151 }
8152 }
8153
8154 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8155 fprintf(stderr, "\n");
8156 }
8157
8158 return ret;
8159 }
8160
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)8161 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
8162 {
8163 const void *handler;
8164 bool do_bailout = 0;
8165
8166 zend_shared_alloc_lock();
8167
8168 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
8169 SHM_UNPROTECT();
8170 zend_jit_unprotect();
8171
8172 zend_try {
8173 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
8174
8175 if (handler) {
8176 zend_jit_link_side_trace(
8177 zend_jit_traces[trace_num].code_start,
8178 zend_jit_traces[trace_num].code_size,
8179 zend_jit_traces[trace_num].jmp_table_size,
8180 exit_num,
8181 handler);
8182 }
8183 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
8184 } zend_catch {
8185 do_bailout = 1;
8186 } zend_end_try();
8187
8188 zend_jit_protect();
8189 SHM_PROTECT();
8190 }
8191
8192 zend_shared_alloc_unlock();
8193
8194 if (do_bailout) {
8195 zend_bailout();
8196 }
8197 }
8198
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)8199 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
8200 {
8201 uint8_t *counter = JIT_G(exit_counters) +
8202 zend_jit_traces[trace_num].exit_counters + exit_num;
8203
8204 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
8205 return 1;
8206 }
8207 (*counter)++;
8208 return 0;
8209 }
8210
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)8211 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
8212 {
8213 uint8_t *counter = JIT_G(exit_counters) +
8214 zend_jit_traces[trace_num].exit_counters + exit_num;
8215
8216 if (*counter + 1 >= JIT_G(hot_side_exit)) {
8217 return 1;
8218 }
8219 (*counter)++;
8220 return 0;
8221 }
8222
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)8223 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
8224 {
8225 zend_jit_trace_stop ret;
8226 const void *handler;
8227 uint8_t orig_trigger;
8228 zend_jit_trace_info *t;
8229 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
8230 bool do_bailout = 0;
8231
8232 zend_shared_alloc_lock();
8233
8234 /* Checks under lock */
8235 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8236 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
8237 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8238 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8239 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8240 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8241 } else {
8242 SHM_UNPROTECT();
8243 zend_jit_unprotect();
8244
8245 zend_try {
8246 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
8247
8248 t->id = ZEND_JIT_TRACE_NUM;
8249 t->root = zend_jit_traces[parent_num].root;
8250 t->parent = parent_num;
8251 t->link = 0;
8252 t->exit_count = 0;
8253 t->child_count = 0;
8254 t->stack_map_size = 0;
8255 t->flags = 0;
8256 t->polymorphism = polymorphism;
8257 t->jmp_table_size = 0;
8258 t->opline = NULL;
8259 t->exit_info = exit_info;
8260 t->stack_map = NULL;
8261 t->consts_count = 0;
8262 t->constants = NULL;
8263
8264 orig_trigger = JIT_G(trigger);
8265 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
8266
8267 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
8268
8269 JIT_G(trigger) = orig_trigger;
8270
8271 if (handler) {
8272 zend_jit_trace_exit_info *shared_exit_info = NULL;
8273
8274 t->exit_info = NULL;
8275 if (t->exit_count) {
8276 /* reallocate exit_info into shared memory */
8277 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
8278 sizeof(zend_jit_trace_exit_info) * t->exit_count);
8279
8280 if (!shared_exit_info) {
8281 if (t->stack_map) {
8282 efree(t->stack_map);
8283 t->stack_map = NULL;
8284 }
8285 if (t->constants) {
8286 efree(t->constants);
8287 t->constants = NULL;
8288 }
8289 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8290 goto exit;
8291 }
8292 memcpy(shared_exit_info, exit_info,
8293 sizeof(zend_jit_trace_exit_info) * t->exit_count);
8294 t->exit_info = shared_exit_info;
8295 }
8296
8297 if (t->stack_map_size) {
8298 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
8299 if (!shared_stack_map) {
8300 efree(t->stack_map);
8301 t->stack_map = NULL;
8302 if (t->constants) {
8303 efree(t->constants);
8304 t->constants = NULL;
8305 }
8306 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8307 goto exit;
8308 }
8309 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
8310 efree(t->stack_map);
8311 t->stack_map = shared_stack_map;
8312 }
8313
8314 if (t->consts_count) {
8315 zend_jit_exit_const *constants = (zend_jit_exit_const*)zend_shared_alloc(t->consts_count * sizeof(zend_jit_exit_const));
8316 if (!constants) {
8317 efree(t->constants);
8318 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8319 goto exit;
8320 }
8321 memcpy(constants, t->constants, t->consts_count * sizeof(zend_jit_exit_const));
8322 efree(t->constants);
8323 t->constants = constants;
8324 }
8325
8326 zend_jit_link_side_trace(
8327 zend_jit_traces[parent_num].code_start,
8328 zend_jit_traces[parent_num].code_size,
8329 zend_jit_traces[parent_num].jmp_table_size,
8330 exit_num,
8331 handler);
8332
8333 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
8334 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
8335
8336 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
8337 ZEND_JIT_TRACE_NUM++;
8338 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
8339
8340 ret = ZEND_JIT_TRACE_STOP_COMPILED;
8341 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
8342 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
8343 if (t->stack_map) {
8344 efree(t->stack_map);
8345 t->stack_map = NULL;
8346 }
8347 if (t->constants) {
8348 efree(t->constants);
8349 t->constants = NULL;
8350 }
8351 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
8352 } else {
8353 if (t->stack_map) {
8354 efree(t->stack_map);
8355 t->stack_map = NULL;
8356 }
8357 if (t->constants) {
8358 efree(t->constants);
8359 t->constants = NULL;
8360 }
8361 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
8362 }
8363
8364 exit:;
8365 } zend_catch {
8366 do_bailout = 1;
8367 } zend_end_try();
8368
8369 zend_jit_protect();
8370 SHM_PROTECT();
8371 }
8372
8373 zend_shared_alloc_unlock();
8374
8375 if (do_bailout) {
8376 zend_bailout();
8377 }
8378
8379 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
8380 && ret == ZEND_JIT_TRACE_STOP_COMPILED
8381 && t->exit_count > 0) {
8382 zend_jit_dump_exit_info(t);
8383 }
8384
8385 return ret;
8386 }
8387
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)8388 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
8389 {
8390 zend_jit_trace_stop stop;
8391 int ret = 0;
8392 uint32_t trace_num;
8393 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8394 uint32_t is_megamorphic = 0;
8395 uint32_t polymorphism = 0;
8396 uint32_t root;
8397 int ret_depth = 0;
8398
8399 trace_num = ZEND_JIT_TRACE_NUM;
8400
8401 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8402 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8403 return 0;
8404 }
8405
8406 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8407 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
8408 trace_num, parent_num, exit_num,
8409 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8410 EX(func)->op_array.scope ? "::" : "",
8411 EX(func)->op_array.function_name ?
8412 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8413 ZSTR_VAL(EX(func)->op_array.filename),
8414 EX(opline)->lineno);
8415 }
8416
8417 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8418 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8419 goto abort;
8420 }
8421
8422 root = zend_jit_traces[parent_num].root;
8423 if (zend_jit_traces[root].child_count >= JIT_G(max_side_traces)) {
8424 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8425 goto abort;
8426 }
8427
8428 if (JIT_G(max_polymorphic_calls) > 0) {
8429 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
8430 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
8431 && EX(call))) {
8432 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
8433 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
8434 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
8435 } else if (!zend_jit_traces[parent_num].polymorphism) {
8436 polymorphism = 1;
8437 } else if (exit_num == 0) {
8438 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
8439 }
8440 }
8441 }
8442
8443 /* Check if this is a side trace of a root LOOP trace */
8444 if ((zend_jit_traces[root].flags & ZEND_JIT_TRACE_LOOP)
8445 && zend_jit_traces[root].op_array != &EX(func)->op_array) {
8446 const zend_op_array *op_array = zend_jit_traces[root].op_array;
8447 const zend_op *opline = zend_jit_traces[root].opline;
8448 zend_jit_op_array_trace_extension *jit_extension =
8449 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8450
8451 if (jit_extension->trace_info[opline - op_array->opcodes].trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8452 zend_execute_data *ex = execute_data;
8453 int n = 0;
8454 do {
8455 ex = ex->prev_execute_data;
8456 n++;
8457 } while (ex && zend_jit_traces[root].op_array != &ex->func->op_array);
8458 if (ex && n <= ZEND_JIT_TRACE_MAX_RET_DEPTH) {
8459 ret_depth = n;
8460 }
8461 }
8462 }
8463
8464 JIT_G(tracing) = 1;
8465 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic, ret_depth);
8466 JIT_G(tracing) = 0;
8467
8468 if (stop & ZEND_JIT_TRACE_HALT) {
8469 ret = -1;
8470 }
8471 stop &= ~ZEND_JIT_TRACE_HALT;
8472
8473 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8474 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8475 const zend_op_array *op_array = trace_buffer[0].op_array;
8476 const zend_op *opline = trace_buffer[1].opline;
8477 zend_jit_op_array_trace_extension *jit_extension =
8478 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8479 size_t offset = jit_extension->offset;
8480
8481 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8482 trace_num,
8483 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8484 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8485 op_array->scope ? "::" : "",
8486 op_array->function_name ?
8487 ZSTR_VAL(op_array->function_name) : "$main",
8488 ZSTR_VAL(op_array->filename),
8489 opline->lineno);
8490 }
8491 }
8492
8493 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8494 zend_jit_dump_trace(trace_buffer, NULL);
8495 }
8496
8497 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8498 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8499 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8500 uint32_t idx = trace_buffer[1].last;
8501 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8502 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8503 trace_num,
8504 link_to);
8505 } else {
8506 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8507 trace_num,
8508 zend_jit_trace_stop_description[stop]);
8509 }
8510 }
8511 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8512 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8513 } else {
8514 const zend_op_array *op_array = trace_buffer[0].op_array;
8515 zend_jit_op_array_trace_extension *jit_extension =
8516 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8517 const zend_op *opline = trace_buffer[1].opline;
8518
8519 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8520 }
8521 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8522 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8523 fprintf(stderr, "---- TRACE %d %s\n",
8524 trace_num,
8525 zend_jit_trace_stop_description[stop]);
8526 }
8527 } else {
8528 goto abort;
8529 }
8530 } else {
8531 abort:
8532 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8533 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8534 trace_num,
8535 zend_jit_trace_stop_description[stop]);
8536 }
8537 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8538 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8539 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8540 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8541 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8542 parent_num, exit_num);
8543 }
8544 }
8545 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8546 execute_data = EG(current_execute_data);
8547 return zend_jit_trace_hot_root(execute_data, EX(opline));
8548 }
8549 }
8550
8551 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8552 fprintf(stderr, "\n");
8553 }
8554
8555 return ret;
8556 }
8557
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8558 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8559 {
8560 uint32_t trace_num = EG(jit_trace_num);
8561 zend_execute_data *execute_data = EG(current_execute_data);
8562 const zend_op *orig_opline = EX(opline);
8563 const zend_op *opline;
8564 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8565 int repeat_last_opline = 0;
8566
8567 /* Deoptimization of VM stack state */
8568 uint32_t i;
8569 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8570 zend_jit_trace_stack *stack = stack_size ? t->stack_map + t->exit_info[exit_num].stack_offset : NULL;
8571
8572 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8573 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8574 call->prev_execute_data = EX(call);
8575 EX(call) = call;
8576 }
8577
8578 for (i = 0; i < stack_size; i++) {
8579 if (STACK_FLAGS(stack, i) == ZREG_CONST) {
8580 if (STACK_TYPE(stack, i) == IS_LONG) {
8581 ZVAL_LONG(EX_VAR_NUM(i), (zend_long)t->constants[STACK_REF(stack, i)].i);
8582 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8583 ZVAL_DOUBLE(EX_VAR_NUM(i), t->constants[STACK_REF(stack, i)].d);
8584 } else {
8585 ZEND_UNREACHABLE();
8586 }
8587 } else if (STACK_FLAGS(stack, i) == ZREG_TYPE_ONLY) {
8588 uint32_t type = STACK_TYPE(stack, i);
8589 if (type <= IS_DOUBLE) {
8590 Z_TYPE_INFO_P(EX_VAR_NUM(i)) = type;
8591 } else {
8592 ZEND_UNREACHABLE();
8593 }
8594 } else if (STACK_FLAGS(stack, i) == ZREG_THIS) {
8595 zend_object *obj = Z_OBJ(EX(This));
8596
8597 GC_ADDREF(obj);
8598 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8599 } else if (STACK_FLAGS(stack, i) == ZREG_ZVAL_ADDREF) {
8600 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8601 } else if (STACK_FLAGS(stack, i) == ZREG_ZVAL_COPY) {
8602 zval *val = (zval*)regs->gpr[STACK_REG(stack, i)];
8603
8604 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8605 /* Undefined array index or property */
8606 const zend_op *op = t->exit_info[exit_num].opline;
8607 ZEND_ASSERT(op);
8608 op--;
8609 if (op->opcode == ZEND_FETCH_DIM_IS || op->opcode == ZEND_FETCH_OBJ_IS) {
8610 ZVAL_NULL(EX_VAR_NUM(i));
8611 } else {
8612 assert(op->opcode == ZEND_FETCH_DIM_R || op->opcode == ZEND_FETCH_LIST_R || op->opcode == ZEND_FETCH_OBJ_R);
8613 repeat_last_opline = 1;
8614 }
8615 } else {
8616 ZVAL_COPY(EX_VAR_NUM(i), val);
8617 }
8618 } else if (STACK_FLAGS(stack, i) & ZREG_SPILL_SLOT) {
8619 ZEND_ASSERT(STACK_REG(stack, i) != ZREG_NONE);
8620 uintptr_t ptr = (uintptr_t)regs->gpr[STACK_REG(stack, i)] + STACK_REF(stack, i);
8621
8622 if (STACK_TYPE(stack, i) == IS_LONG) {
8623 ZVAL_LONG(EX_VAR_NUM(i), *(zend_long*)ptr);
8624 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8625 ZVAL_DOUBLE(EX_VAR_NUM(i), *(double*)ptr);
8626 } else {
8627 ZEND_UNREACHABLE();
8628 }
8629 } else if (STACK_REG(stack, i) != ZREG_NONE) {
8630 if (STACK_TYPE(stack, i) == IS_LONG) {
8631 zend_long val = regs->gpr[STACK_REG(stack, i)];
8632 ZVAL_LONG(EX_VAR_NUM(i), val);
8633 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8634 double val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8635 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8636 } else {
8637 ZEND_UNREACHABLE();
8638 }
8639 }
8640 }
8641
8642 if (repeat_last_opline) {
8643 EX(opline) = t->exit_info[exit_num].opline - 1;
8644 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8645 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8646 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8647 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8648 }
8649 return 1;
8650 }
8651
8652 opline = t->exit_info[exit_num].opline;
8653
8654 if (opline) {
8655 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8656 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8657 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8658 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8659 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8660 EX(opline) = opline-1;
8661 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8662 }
8663 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8664 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8665 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8666 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8667 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8668 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8669 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8670 EX(opline) = opline-1;
8671 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8672 }
8673 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8674 if (EG(exception)) {
8675 return 1;
8676 }
8677 }
8678 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8679 ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0);
8680 zend_function *func = (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg];
8681
8682 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8683 zend_string_release_ex(func->common.function_name, 0);
8684 zend_free_trampoline(func);
8685 EX(opline) = opline;
8686 return 1;
8687 }
8688 }
8689
8690 /* Set VM opline to continue interpretation */
8691 EX(opline) = opline;
8692 }
8693
8694 if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) {
8695 return 1;
8696 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8697 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8698 return 0;
8699 }
8700
8701 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8702 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8703 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8704
8705 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8706 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8707 trace_num,
8708 exit_num,
8709 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8710 EX(func)->op_array.scope ? "::" : "",
8711 EX(func)->op_array.function_name ?
8712 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8713 ZSTR_VAL(EX(func)->op_array.filename),
8714 EX(opline)->lineno);
8715 }
8716
8717 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8718 zend_jit_op_array_trace_extension *jit_extension;
8719 uint32_t num = trace_num;
8720
8721 while (t->root != num) {
8722 num = t->root;
8723 t = &zend_jit_traces[num];
8724 }
8725
8726 zend_shared_alloc_lock();
8727
8728 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8729
8730 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8731 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8732 /* skip: not JIT-ed nor blacklisted */
8733 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8734 /* too many root traces, blacklist the root trace */
8735 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
8736 SHM_UNPROTECT();
8737 zend_jit_unprotect();
8738
8739 ((zend_op*)opline)->handler =
8740 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->orig_handler;
8741
8742 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= ~ZEND_JIT_TRACE_JITED;
8743 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
8744
8745 zend_jit_protect();
8746 SHM_PROTECT();
8747 }
8748 } else {
8749 SHM_UNPROTECT();
8750 zend_jit_unprotect();
8751
8752 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8753 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8754 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8755 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8756 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8757 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8758 }
8759 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8760 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8761
8762 zend_jit_protect();
8763 SHM_PROTECT();
8764 }
8765
8766 zend_shared_alloc_unlock();
8767
8768 return 0;
8769 }
8770
8771 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8772 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8773 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8774 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8775 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8776 trace_num, exit_num);
8777 }
8778 return 0;
8779 }
8780 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8781 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8782 }
8783
8784 /* Return 1 to call original handler instead of the same JIT-ed trace */
8785 return (orig_opline == t->opline && EX(opline) == orig_opline);
8786 }
8787
zend_jit_trace_supported(const zend_op * opline)8788 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8789 {
8790 switch (opline->opcode) {
8791 case ZEND_CATCH:
8792 case ZEND_FAST_CALL:
8793 case ZEND_FAST_RET:
8794 return ZEND_JIT_TRACE_UNSUPPORTED;
8795 default:
8796 return ZEND_JIT_TRACE_SUPPORTED;
8797 }
8798 }
8799
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8800 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8801 {
8802 zend_jit_op_array_trace_extension *jit_extension;
8803 uint32_t i;
8804
8805 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8806 for (i = 0; i < op_array->last; i++) {
8807 jit_extension->trace_info[i].trace_flags &=
8808 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8809 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8810 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8811 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8812 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8813 } else {
8814 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8815 }
8816 }
8817 return SUCCESS;
8818 }
8819
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8820 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8821 {
8822 zend_op *opline;
8823 zend_jit_op_array_trace_extension *jit_extension;
8824 uint32_t i;
8825
8826 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8827
8828 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8829 if (!jit_extension) {
8830 return FAILURE;
8831 }
8832 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8833 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8834 jit_extension->op_array = op_array;
8835 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8836 for (i = 0; i < op_array->last; i++) {
8837 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8838 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8839 jit_extension->trace_info[i].counter = NULL;
8840 jit_extension->trace_info[i].trace_flags =
8841 zend_jit_trace_supported(&op_array->opcodes[i]);
8842 }
8843 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8844
8845 if (JIT_G(hot_loop)) {
8846 zend_cfg cfg;
8847
8848 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8849
8850 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8851 return FAILURE;
8852 }
8853
8854 for (i = 0; i < cfg.blocks_count; i++) {
8855 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8856 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8857 /* loop header */
8858 opline = op_array->opcodes + cfg.blocks[i].start;
8859 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8860 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8861 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8862 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8863 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8864 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8865 }
8866 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8867 ZEND_JIT_TRACE_START_LOOP;
8868 }
8869 }
8870 }
8871 }
8872 }
8873
8874 if (JIT_G(hot_func)) {
8875 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8876 opline = op_array->opcodes;
8877 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8878 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8879 opline++;
8880 }
8881 }
8882
8883 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8884 /* function entry */
8885 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8886 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8887 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8888 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8889 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8890 ZEND_JIT_TRACE_START_ENTER;
8891 }
8892 }
8893
8894 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8895
8896 return SUCCESS;
8897 }
8898
zend_jit_trace_init_caches(void)8899 static void zend_jit_trace_init_caches(void)
8900 {
8901 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8902 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8903 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8904 JIT_G(bad_root_slot) = 0;
8905
8906 if (JIT_G(exit_counters)) {
8907 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8908 }
8909 }
8910
zend_jit_trace_reset_caches(void)8911 static void zend_jit_trace_reset_caches(void)
8912 {
8913 JIT_G(tracing) = 0;
8914 #ifdef ZTS
8915 if (!JIT_G(exit_counters)) {
8916 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8917 }
8918 #endif
8919 }
8920
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8921 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8922 {
8923 if (jit_globals->exit_counters) {
8924 free(jit_globals->exit_counters);
8925 }
8926 }
8927
zend_jit_trace_restart(void)8928 static void zend_jit_trace_restart(void)
8929 {
8930 ZEND_JIT_TRACE_NUM = 1;
8931 ZEND_JIT_COUNTER_NUM = 0;
8932 ZEND_JIT_EXIT_NUM = 0;
8933 ZEND_JIT_EXIT_COUNTERS = 0;
8934 ZCSG(jit_counters_stopped) = false;
8935
8936 zend_jit_trace_init_caches();
8937 }
8938
